diff --git a/.env.example b/.env.example index e69de29..713031f 100644 --- a/.env.example +++ b/.env.example @@ -0,0 +1,4 @@ +LOCAL_ONLY=True +SECRET_KEY= +UPLOADS_DIR=./uploads +PROCESSED_DIR=./processed \ No newline at end of file diff --git a/.gitignore b/.gitignore index 51116de..fd4c6f8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,7 @@ core huey.* *.db req.txt +settings.yml +app.log +/config/* diff --git a/docker-compose.yml b/docker-compose.yml index 809837f..039173e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,12 +5,18 @@ services: build: . env_file: - .env + environment: + - LOCAL_ONLY=True # set to False to enable OIDC auth (requires configuration in settings.yml) + - SECRET_KEY= # if using auth + - UPLOADS_DIR=/app/uploads + - PROCESSED_DIR=/app/processed #user: "1000:1000" ports: - "6969:8000" volumes: # Mount local directories and files into the container for persistence + - ./config:/app/config - ./uploads_data:/app/uploads - ./processed_data:/app/processed diff --git a/main.py b/main.py index 6f678fd..057f72f 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,5 @@ +# main.py (merged) + import logging import shutil import subprocess @@ -5,10 +7,15 @@ import traceback import uuid import shlex import yaml +import os from contextlib import asynccontextmanager from datetime import datetime, timezone from pathlib import Path from typing import Dict, List, Any +import resource +from threading import Semaphore +from logging.handlers import RotatingFileHandler +from urllib.parse import urlencode import ocrmypdf import pypdf @@ -17,11 +24,11 @@ from PIL import Image from faster_whisper import WhisperModel from fastapi import (Depends, FastAPI, File, Form, HTTPException, Request, UploadFile, status, Body) -from fastapi.responses import FileResponse, JSONResponse +from fastapi.responses import FileResponse, JSONResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from huey import SqliteHuey -from pydantic import BaseModel, ConfigDict, field_serializer # MODIFIED: Import field_serializer +from pydantic import BaseModel, ConfigDict, field_serializer from sqlalchemy import (Column, DateTime, Integer, String, Text, create_engine, delete, event) from sqlalchemy.orm import Session, declarative_base, sessionmaker @@ -29,43 +36,104 @@ from sqlalchemy.pool import NullPool from string import Formatter from werkzeug.utils import secure_filename from typing import List as TypingList +from starlette.middleware.sessions import SessionMiddleware +from authlib.integrations.starlette_client import OAuth +from dotenv import load_dotenv + +load_dotenv() +# Instantiate OAuth object (was referenced in code) +oauth = OAuth() + + # -------------------------------------------------------------------------------- -# --- 1. CONFIGURATION +# --- 1. CONFIGURATION & SECURITY HELPERS # -------------------------------------------------------------------------------- +# --- Path Safety --- +UPLOADS_BASE = Path(os.environ.get("UPLOADS_DIR", "/app/uploads")).resolve() +PROCESSED_BASE = Path(os.environ.get("PROCESSED_DIR", "/app/processed")).resolve() +CHUNK_TMP_BASE = Path(os.environ.get("CHUNK_TMP_DIR", str(UPLOADS_BASE / "tmp"))).resolve() + +def ensure_path_is_safe(p: Path, allowed_bases: List[Path]): + """Ensure a path resolves to a location within one of the allowed base directories.""" + try: + resolved_p = p.resolve() + if not any(resolved_p.is_relative_to(base) for base in allowed_bases): + raise ValueError(f"Path {resolved_p} is outside of allowed directories.") + return resolved_p + except Exception as e: + logger = logging.getLogger(__name__) + logger.error(f"Path safety check failed for {p}: {e}") + raise ValueError("Invalid or unsafe path specified.") + +# --- Resource Limiting --- +def _limit_resources_preexec(): + """Set resource limits for child processes to prevent DoS attacks.""" + try: + # 6000s CPU, 2GB address space, i dont know if thats too much tbh + resource.setrlimit(resource.RLIMIT_CPU, (6000, 6000)) + resource.setrlimit(resource.RLIMIT_AS, (4 * 1024 * 1024 * 1024, 2 * 1024 * 1024 * 1024)) + except Exception as e: + # This may fail in some environments (e.g. Windows, some containers) + logging.getLogger(__name__).warning(f"Could not set resource limits: {e}") + pass + +# --- Model concurrency semaphore --- +MODEL_CONCURRENCY = int(os.environ.get("MODEL_CONCURRENCY", "1")) +_model_semaphore = Semaphore(MODEL_CONCURRENCY) + +# --- Logging Setup --- logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +_log_handler = RotatingFileHandler("app.log", maxBytes=10*1024*1024, backupCount=5) +_log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s %(message)s') +_log_handler.setFormatter(_log_formatter) +logging.getLogger().addHandler(_log_handler) logger = logging.getLogger(__name__) +# --- Environment Mode --- +LOCAL_ONLY_MODE = os.getenv('LOCAL_ONLY', 'True').lower() in ('true', '1', 't') +if LOCAL_ONLY_MODE: + logger.warning("Authentication is DISABLED. Running in LOCAL_ONLY mode.") class AppPaths(BaseModel): BASE_DIR: Path = Path(__file__).resolve().parent - UPLOADS_DIR: Path = BASE_DIR / "uploads" - PROCESSED_DIR: Path = BASE_DIR / "processed" + UPLOADS_DIR: Path = UPLOADS_BASE + PROCESSED_DIR: Path = PROCESSED_BASE + CHUNK_TMP_DIR: Path = CHUNK_TMP_BASE DATABASE_URL: str = f"sqlite:///{BASE_DIR / 'jobs.db'}" HUEY_DB_PATH: str = str(BASE_DIR / "huey.db") - SETTINGS_FILE: Path = BASE_DIR / "settings.yml" + CONFIG_DIR: Path = BASE_DIR / "config" + SETTINGS_FILE: Path = CONFIG_DIR / "settings.yml" + DEFAULT_SETTINGS_FILE: Path = BASE_DIR / "settings.default.yml" PATHS = AppPaths() APP_CONFIG: Dict[str, Any] = {} -PATHS.UPLOADS_DIR.mkdir(exist_ok=True) -PATHS.PROCESSED_DIR.mkdir(exist_ok=True) +PATHS.UPLOADS_DIR.mkdir(exist_ok=True, parents=True) +PATHS.PROCESSED_DIR.mkdir(exist_ok=True, parents=True) +PATHS.CHUNK_TMP_DIR.mkdir(exist_ok=True, parents=True) +PATHS.CONFIG_DIR.mkdir(exist_ok=True, parents=True) def load_app_config(): + """ + Loads configuration from settings.yml, with a fallback to settings.default.yml, + and finally to hardcoded defaults if both files are missing. + """ global APP_CONFIG try: + # --- Primary Method: Attempt to load settings.yml --- with open(PATHS.SETTINGS_FILE, 'r', encoding='utf8') as f: cfg_raw = yaml.safe_load(f) or {} - # basic defaults + + # This logic block is intentionally duplicated to maintain compatibility defaults = { "app_settings": {"max_file_size_mb": 100, "allowed_all_extensions": []}, - "transcription_settings": {"whisper": {"allowed_models": ["tiny", "base", "small"], "compute_type": "int8"}}, + "transcription_settings": {"whisper": {"allowed_models": ["tiny", "base", "small"], "compute_type": "int8", "device": "cpu"}}, "conversion_tools": {}, - "ocr_settings": {"ocrmypdf": {}} + "ocr_settings": {"ocrmypdf": {}}, + "auth_settings": {"oidc_client_id": "", "oidc_client_secret": "", "oidc_server_metadata_url": "", "admin_users": []} } - # shallow merge (safe for top-level keys) cfg = defaults.copy() - cfg.update(cfg_raw) - # normalize app settings + cfg.update(cfg_raw) # Merge loaded settings into defaults app_settings = cfg.get("app_settings", {}) max_mb = app_settings.get("max_file_size_mb", 100) app_settings["max_file_size_bytes"] = int(max_mb) * 1024 * 1024 @@ -76,16 +144,45 @@ def load_app_config(): cfg["app_settings"] = app_settings APP_CONFIG = cfg logger.info("Successfully loaded settings from settings.yml") - except (FileNotFoundError, yaml.YAMLError) as e: - logging.getLogger(__name__).exception(f"Could not load settings.yml: {e}. Using defaults.") - - APP_CONFIG = { - "app_settings": {"max_file_size_mb": 100, "max_file_size_bytes": 100 * 1024 * 1024, "allowed_all_extensions": set()}, - "transcription_settings": {"whisper": {"allowed_models": ["tiny", "base", "small"], "compute_type": "int8"}}, - "conversion_tools": {}, - "ocr_settings": {"ocrmypdf": {}} - } + except (FileNotFoundError, yaml.YAMLError) as e: + logger.warning(f"Could not load settings.yml: {e}. Falling back to settings.default.yml...") + try: + # --- Fallback Method: Attempt to load settings.default.yml --- + with open(PATHS.DEFAULT_SETTINGS_FILE, 'r', encoding='utf8') as f: + cfg_raw = yaml.safe_load(f) or {} + + # The same processing logic is applied to the fallback file + defaults = { + "app_settings": {"max_file_size_mb": 100, "allowed_all_extensions": []}, + "transcription_settings": {"whisper": {"allowed_models": ["tiny", "base", "small"], "compute_type": "int8", "device": "cpu"}}, + "conversion_tools": {}, + "ocr_settings": {"ocrmypdf": {}}, + "auth_settings": {"oidc_client_id": "", "oidc_client_secret": "", "oidc_server_metadata_url": "", "admin_users": []} + } + cfg = defaults.copy() + cfg.update(cfg_raw) # Merge loaded settings into defaults + app_settings = cfg.get("app_settings", {}) + max_mb = app_settings.get("max_file_size_mb", 100) + app_settings["max_file_size_bytes"] = int(max_mb) * 1024 * 1024 + allowed = app_settings.get("allowed_all_extensions", []) + if not isinstance(allowed, (list, set)): + allowed = list(allowed) + app_settings["allowed_all_extensions"] = set(allowed) + cfg["app_settings"] = app_settings + APP_CONFIG = cfg + logger.info("Successfully loaded settings from settings.default.yml") + + except (FileNotFoundError, yaml.YAMLError) as e_fallback: + # --- Final Failsafe: Use hardcoded defaults --- + logger.error(f"CRITICAL: Fallback file settings.default.yml also failed: {e_fallback}. Using hardcoded defaults.") + APP_CONFIG = { + "app_settings": {"max_file_size_mb": 100, "max_file_size_bytes": 100 * 1024 * 1024, "allowed_all_extensions": set()}, + "transcription_settings": {"whisper": {"allowed_models": ["tiny", "base", "small"], "compute_type": "int8", "device": "cpu"}}, + "conversion_tools": {}, + "ocr_settings": {"ocrmypdf": {}}, + "auth_settings": {"oidc_client_id": "", "oidc_client_secret": "", "oidc_server_metadata_url": "", "admin_users": []} + } # -------------------------------------------------------------------------------- @@ -101,10 +198,6 @@ Base = declarative_base() @event.listens_for(engine, "connect") def _set_sqlite_pragmas(dbapi_connection, connection_record): - """ - Enable WAL mode and set sane synchronous for better concurrency - between the FastAPI process and Huey worker processes. - """ c = dbapi_connection.cursor() try: c.execute("PRAGMA journal_mode=WAL;") @@ -115,6 +208,7 @@ def _set_sqlite_pragmas(dbapi_connection, connection_record): class Job(Base): __tablename__ = "jobs" id = Column(String, primary_key=True, index=True) + user_id = Column(String, index=True, nullable=True) task_type = Column(String, index=True) status = Column(String, default="pending") progress = Column(Integer, default=0) @@ -137,6 +231,7 @@ def get_db(): class JobCreate(BaseModel): id: str + user_id: str | None = None task_type: str original_filename: str input_filepath: str @@ -157,21 +252,29 @@ class JobSchema(BaseModel): created_at: datetime updated_at: datetime model_config = ConfigDict(from_attributes=True) - - # NEW: This serializer ensures the datetime string sent to the frontend ALWAYS - # includes the 'Z' UTC indicator, fixing the timezone bug. @field_serializer('created_at', 'updated_at') def serialize_dt(self, dt: datetime, _info): return dt.isoformat() + "Z" +class FinalizeUploadPayload(BaseModel): + upload_id: str + original_filename: str + total_chunks: int + task_type: str + model_size: str = "" + output_format: str = "" + # -------------------------------------------------------------------------------- # --- 3. CRUD OPERATIONS # -------------------------------------------------------------------------------- def get_job(db: Session, job_id: str): return db.query(Job).filter(Job.id == job_id).first() -def get_jobs(db: Session, skip: int = 0, limit: int = 100): - return db.query(Job).order_by(Job.created_at.desc()).offset(skip).limit(limit).all() +def get_jobs(db: Session, user_id: str | None = None, skip: int = 0, limit: int = 100): + query = db.query(Job) + if user_id: + query = query.filter(Job.user_id == user_id) + return query.order_by(Job.created_at.desc()).offset(skip).limit(limit).all() def create_job(db: Session, job: JobCreate): db_job = Job(**job.model_dump()) @@ -209,88 +312,62 @@ def mark_job_as_completed(db: Session, job_id: str, output_filepath_str: str | N db.commit() return db_job -# ... (The rest of the file is unchanged and remains the same) ... - # -------------------------------------------------------------------------------- # --- 4. BACKGROUND TASK SETUP # -------------------------------------------------------------------------------- huey = SqliteHuey(filename=PATHS.HUEY_DB_PATH) - -# Whisper model cache per worker process WHISPER_MODELS_CACHE: Dict[str, WhisperModel] = {} def get_whisper_model(model_size: str, whisper_settings: dict) -> WhisperModel: if model_size in WHISPER_MODELS_CACHE: - logger.info(f"Found model '{model_size}' in cache. Reusing.") + logger.info(f"Reusing cached model '{model_size}'.") return WHISPER_MODELS_CACHE[model_size] - device = whisper_settings.get("device", "cpu") - compute_type = whisper_settings.get('compute_type', 'int8') - logger.info(f"Whisper model '{model_size}' not in cache. Loading into memory on device={device}...") - try: - model = WhisperModel(model_size, device=device, compute_type=compute_type) - except Exception: - logger.exception("Failed to load whisper model") - raise - WHISPER_MODELS_CACHE[model_size] = model - logger.info(f"Model '{model_size}' loaded successfully.") - return model + with _model_semaphore: + if model_size in WHISPER_MODELS_CACHE: + return WHISPER_MODELS_CACHE[model_size] + logger.info(f"Loading Whisper model '{model_size}'...") + model = WhisperModel(model_size, device=whisper_settings.get("device", "cpu"), compute_type=whisper_settings.get('compute_type', 'int8')) + WHISPER_MODELS_CACHE[model_size] = model + logger.info(f"Model '{model_size}' loaded.") + return model -# Helper: safe run_command (trimmed logs + timeout) def run_command(argv: TypingList[str], timeout: int = 300): try: - res = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=timeout) + res = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=timeout, preexec_fn=_limit_resources_preexec) + if res.returncode != 0: + raise Exception(f"Command failed with exit code {res.returncode}. Stderr: {res.stderr[:1000]}") + return res except subprocess.TimeoutExpired: raise Exception(f"Command timed out after {timeout}s") - if res.returncode != 0: - stderr = (res.stderr or "")[:4000] - stdout = (res.stdout or "")[:4000] - raise Exception(f"Command failed exit {res.returncode}. stderr: {stderr}; stdout: {stdout}") - return res - -# Helper: validate and build command from template with allowlist -ALLOWED_VARS = {"input", "output", "output_dir", "output_ext", "quality", "speed", "preset", "device", "dpi", "samplerate", "bitdepth", "filter"} def validate_and_build_command(template_str: str, mapping: Dict[str, str]) -> TypingList[str]: - """ - Validate placeholders against ALLOWED_VARS and build a safe argv list. - If a template uses allowed placeholders that are missing from `mapping`, - auto-fill sensible defaults: - - 'filter' -> mapping.get('output_ext', '') - - others -> empty string - This prevents KeyError while preserving the allowlist security check. - """ fmt = Formatter() used = {fname for _, fname, _, _ in fmt.parse(template_str) if fname} + ALLOWED_VARS = {"input", "output", "output_dir", "output_ext", "quality", "speed", "preset", "device", "dpi", "samplerate", "bitdepth", "filter"} bad = used - ALLOWED_VARS if bad: raise ValueError(f"Command template contains disallowed placeholders: {bad}") - - # auto-fill missing allowed placeholders with safe defaults - safe_mapping = dict(mapping) # shallow copy to avoid mutating caller mapping + safe_mapping = dict(mapping) for name in used: if name not in safe_mapping: - if name == "filter": - safe_mapping[name] = safe_mapping.get("output_ext", "") - else: - safe_mapping[name] = "" - + safe_mapping[name] = safe_mapping.get("output_ext", "") if name == "filter" else "" formatted = template_str.format(**safe_mapping) return shlex.split(formatted) @huey.task() def run_transcription_task(job_id: str, input_path_str: str, output_path_str: str, model_size: str, whisper_settings: dict): db = SessionLocal() + input_path = Path(input_path_str) try: job = get_job(db, job_id) - if not job or job.status == 'cancelled': - return + if not job or job.status == 'cancelled': return update_job_status(db, job_id, "processing") model = get_whisper_model(model_size, whisper_settings) logger.info(f"Starting transcription for job {job_id}") - segments, info = model.transcribe(input_path_str, beam_size=5) + segments, info = model.transcribe(str(input_path), beam_size=5) full_transcript = [] for segment in segments: - job_check = get_job(db, job_id) # Check for cancellation during long tasks + job_check = get_job(db, job_id) if job_check.status == 'cancelled': logger.info(f"Job {job_id} cancelled during transcription.") return @@ -299,7 +376,6 @@ def run_transcription_task(job_id: str, input_path_str: str, output_path_str: st update_job_status(db, job_id, "processing", progress=progress) full_transcript.append(segment.text.strip()) transcript_text = "\n".join(full_transcript) - # atomic write of transcript — keep the real extension and mark tmp in the name out_path = Path(output_path_str) tmp_out = out_path.with_name(f"{out_path.stem}.tmp-{uuid.uuid4().hex}{out_path.suffix}") with tmp_out.open("w", encoding="utf-8") as f: @@ -307,23 +383,29 @@ def run_transcription_task(job_id: str, input_path_str: str, output_path_str: st tmp_out.replace(out_path) mark_job_as_completed(db, job_id, output_filepath_str=output_path_str, preview=transcript_text) logger.info(f"Transcription for job {job_id} completed.") - except Exception: + except Exception as e: logger.exception(f"ERROR during transcription for job {job_id}") - update_job_status(db, job_id, "failed", error="See server logs for details.") + update_job_status(db, job_id, "failed", error=f"Transcription failed: {e}") finally: - Path(input_path_str).unlink(missing_ok=True) + try: + ensure_path_is_safe(input_path, [PATHS.UPLOADS_DIR, PATHS.CHUNK_TMP_DIR]) + input_path.unlink(missing_ok=True) + except Exception: + # swallow cleanup errors but log + logger.exception("Failed to cleanup input file after transcription.") db.close() @huey.task() def run_pdf_ocr_task(job_id: str, input_path_str: str, output_path_str: str, ocr_settings: dict): db = SessionLocal() + input_path = Path(input_path_str) try: job = get_job(db, job_id) if not job or job.status == 'cancelled': return update_job_status(db, job_id, "processing") logger.info(f"Starting PDF OCR for job {job_id}") - ocrmypdf.ocr(input_path_str, output_path_str, + ocrmypdf.ocr(str(input_path), str(output_path_str), deskew=ocr_settings.get('deskew', True), force_ocr=ocr_settings.get('force_ocr', True), clean=ocr_settings.get('clean', True), @@ -334,42 +416,52 @@ def run_pdf_ocr_task(job_id: str, input_path_str: str, output_path_str: str, ocr preview = "\n".join(page.extract_text() or "" for page in reader.pages) mark_job_as_completed(db, job_id, output_filepath_str=output_path_str, preview=preview) logger.info(f"PDF OCR for job {job_id} completed.") - except Exception: + except Exception as e: logger.exception(f"ERROR during PDF OCR for job {job_id}") - update_job_status(db, job_id, "failed", error="See server logs for details.") + update_job_status(db, job_id, "failed", error=f"PDF OCR failed: {e}") finally: - Path(input_path_str).unlink(missing_ok=True) + try: + ensure_path_is_safe(input_path, [PATHS.UPLOADS_DIR]) + input_path.unlink(missing_ok=True) + except Exception: + logger.exception("Failed to cleanup input file after PDF OCR.") db.close() @huey.task() def run_image_ocr_task(job_id: str, input_path_str: str, output_path_str: str): db = SessionLocal() + input_path = Path(input_path_str) try: job = get_job(db, job_id) if not job or job.status == 'cancelled': return update_job_status(db, job_id, "processing", progress=50) logger.info(f"Starting Image OCR for job {job_id}") - text = pytesseract.image_to_string(Image.open(input_path_str)) - # atomic write of OCR text + text = pytesseract.image_to_string(Image.open(str(input_path))) out_path = Path(output_path_str) - tmp_out = out_path.with_name(f"{out_path.stem}.tmp-{uuid.uuid4().hex}{out_path.suffix}") + tmp_out = out_path.with_name(f"{out_path.stem}.tmp-{uuid.uuid4().hex}{out_path.suffix}") with tmp_out.open("w", encoding="utf-8") as f: f.write(text) tmp_out.replace(out_path) mark_job_as_completed(db, job_id, output_filepath_str=output_path_str, preview=text) logger.info(f"Image OCR for job {job_id} completed.") - except Exception: + except Exception as e: logger.exception(f"ERROR during Image OCR for job {job_id}") - update_job_status(db, job_id, "failed", error="See server logs for details.") + update_job_status(db, job_id, "failed", error=f"Image OCR failed: {e}") finally: - Path(input_path_str).unlink(missing_ok=True) + try: + ensure_path_is_safe(input_path, [PATHS.UPLOADS_DIR]) + input_path.unlink(missing_ok=True) + except Exception: + logger.exception("Failed to cleanup input file after Image OCR.") db.close() @huey.task() def run_conversion_task(job_id: str, input_path_str: str, output_path_str: str, tool: str, task_key: str, conversion_tools_config: dict): db = SessionLocal() + input_path = Path(input_path_str) + output_path = Path(output_path_str) temp_input_file = None temp_output_file = None try: @@ -381,8 +473,7 @@ def run_conversion_task(job_id: str, input_path_str: str, output_path_str: str, tool_config = conversion_tools_config.get(tool) if not tool_config: raise ValueError(f"Unknown conversion tool: {tool}") - input_path = Path(input_path_str) - output_path = Path(output_path_str) + current_input_path = input_path # Pre-processing for specific tools @@ -399,7 +490,6 @@ def run_conversion_task(job_id: str, input_path_str: str, output_path_str: str, update_job_status(db, job_id, "processing", progress=50) # prepare temporary output and mapping - # use a temp filename that preserves the real extension, e.g. file.tmp-.pdf temp_output_file = output_path.with_name(f"{output_path.stem}.tmp-{uuid.uuid4().hex}{output_path.suffix}") mapping = { "input": str(current_input_path), @@ -410,7 +500,10 @@ def run_conversion_task(job_id: str, input_path_str: str, output_path_str: str, # tool specific mapping adjustments if tool.startswith("ghostscript"): - device, setting = task_key.split('_') + # task_key form: "device_setting" + parts = task_key.split('_', 1) + device = parts[0] if parts else "" + setting = parts[1] if len(parts) > 1 else "" mapping.update({"device": device, "dpi": setting, "preset": setting}) elif tool == "pngquant": _, quality_key = task_key.split('_') @@ -418,17 +511,22 @@ def run_conversion_task(job_id: str, input_path_str: str, output_path_str: str, speed_map = {"hq": "1", "mq": "3", "fast": "11"} mapping.update({"quality": quality_map.get(quality_key, "65-80"), "speed": speed_map.get(quality_key, "3")}) elif tool == "sox": - _, rate, depth = task_key.split('_') + rate, depth = '', '' + try: + _, rate, depth = task_key.split('_') + depth = ('-b' + depth.replace('b', '')) if 'b' in depth else '16b' + except: + _, rate = task_key.split('_') + depth = '' + rate = rate.replace('k', '000') if 'k' in rate else rate - depth = depth.replace('b', '') if 'b' in depth else '16' - mapping.update({"samplerate": rate, "bitdepth": depth}) + mapping.update({"samplerate": rate, "bitdepth": depth}) elif tool == "mozjpeg": _, quality = task_key.split('_') quality = quality.replace('q', '') mapping.update({"quality": quality}) elif tool == "libreoffice": target_ext = output_path.suffix.lstrip('.') - # tool_config may include a 'filters' mapping (see settings.yml example) filter_val = tool_config.get("filters", {}).get(target_ext, target_ext) mapping["filter"] = filter_val @@ -436,28 +534,36 @@ def run_conversion_task(job_id: str, input_path_str: str, output_path_str: str, command = validate_and_build_command(command_template_str, mapping) logger.info(f"Executing command: {' '.join(command)}") - # execute command with timeout and trimmed logs on error result = run_command(command, timeout=tool_config.get("timeout", 300)) - # handle LibreOffice special case: sometimes it writes differently - # Special-case LibreOffice: support per-format export filters via settings.yml - - - # move temp output into final location atomically if temp_output_file and temp_output_file.exists(): temp_output_file.replace(output_path) - mark_job_as_completed(db, job_id, output_filepath_str=output_path_str, preview=f"Successfully converted file.") + mark_job_as_completed(db, job_id, output_filepath_str=str(output_path), preview=f"Successfully converted file.") logger.info(f"Conversion for job {job_id} completed.") - except Exception: + except Exception as e: logger.exception(f"ERROR during conversion for job {job_id}") - update_job_status(db, job_id, "failed", error="See server logs for details.") + update_job_status(db, job_id, "failed", error=f"Conversion failed: {e}") finally: - Path(input_path_str).unlink(missing_ok=True) + try: + ensure_path_is_safe(input_path, [PATHS.UPLOADS_DIR]) + input_path.unlink(missing_ok=True) + except Exception: + logger.exception("Failed to cleanup main input file after conversion.") if temp_input_file: - temp_input_file.unlink(missing_ok=True) + try: + temp_input_file_path = Path(temp_input_file) + ensure_path_is_safe(temp_input_file_path, [PATHS.UPLOADS_DIR, PATHS.PROCESSED_DIR]) + temp_input_file_path.unlink(missing_ok=True) + except Exception: + logger.exception("Failed to cleanup temp input file after conversion.") if temp_output_file: - temp_output_file.unlink(missing_ok=True) + try: + temp_output_file_path = Path(temp_output_file) + ensure_path_is_safe(temp_output_file_path, [PATHS.UPLOADS_DIR, PATHS.PROCESSED_DIR]) + temp_output_file_path.unlink(missing_ok=True) + except Exception: + logger.exception("Failed to cleanup temp output file after conversion.") db.close() # -------------------------------------------------------------------------------- @@ -468,20 +574,166 @@ async def lifespan(app: FastAPI): logger.info("Application starting up...") Base.metadata.create_all(bind=engine) load_app_config() + ENV = os.environ.get('ENV', 'dev').lower() # probably reduntant because I load the .env at the start but whatever + ALLOW_LOCAL_ONLY = os.environ.get('ALLOW_LOCAL_ONLY', 'false').lower() == 'true' + if LOCAL_ONLY_MODE and ENV != 'dev' and not ALLOW_LOCAL_ONLY: + raise RuntimeError('LOCAL_ONLY_MODE may only be enabled in dev or when ALLOW_LOCAL_ONLY=true is set.') + if not LOCAL_ONLY_MODE: + oidc_cfg = APP_CONFIG.get('auth_settings', {}) + if not all(oidc_cfg.get(k) for k in ['oidc_client_id', 'oidc_client_secret', 'oidc_server_metadata_url']): + logger.error("OIDC auth settings are incomplete. Auth will be disabled if not in LOCAL_ONLY_MODE.") + else: + oauth.register( + name='oidc', + client_id=oidc_cfg.get('oidc_client_id'), + client_secret=oidc_cfg.get('oidc_client_secret'), + server_metadata_url=oidc_cfg.get('oidc_server_metadata_url'), + client_kwargs={'scope': 'openid email profile'}, + userinfo_endpoint=oidc_cfg.get('oidc_userinfo_endpoint'), + end_session_endpoint=oidc_cfg.get('oidc_end_session_endpoint') + ) + logger.info('OAuth registered.') yield - logger.info("Application shutting down...") + logger.info('Application shutting down...') app = FastAPI(lifespan=lifespan) -app.mount("/static", StaticFiles(directory=PATHS.BASE_DIR / "static"), name="static") -templates = Jinja2Templates(directory=PATHS.BASE_DIR / "templates") +ENV = os.environ.get('ENV', 'dev').lower() +SECRET_KEY = os.environ.get('SECRET_KEY') +if not SECRET_KEY and not LOCAL_ONLY_MODE and ENV != 'dev': + raise RuntimeError('SECRET_KEY must be set in production when authentication is enabled.') +if not SECRET_KEY: + logger.warning('SECRET_KEY is not set. Generating a temporary key. Sessions will not persist across restarts.') + SECRET_KEY = os.urandom(24).hex() +# Should probably set https_only=True in production behind HTTPS i guess +app.add_middleware( + SessionMiddleware, + secret_key=SECRET_KEY, + https_only=False, + same_site='lax', + max_age=14 * 24 * 60 * 60 # 14 days in seconds +) + + +# Static / templates +app.mount("/static", StaticFiles(directory=str(PATHS.BASE_DIR / "static")), name="static") +templates = Jinja2Templates(directory=str(PATHS.BASE_DIR / "templates")) + +# --- AUTH & USER HELPERS --- +def get_current_user(request: Request): + if LOCAL_ONLY_MODE: + return {'sub': 'local_user', 'email': 'local@user.com', 'name': 'Local User'} + return request.session.get('user') + +def is_admin(request: Request) -> bool: + if LOCAL_ONLY_MODE: return True + user = get_current_user(request) + if not user: return False + admin_users = APP_CONFIG.get("auth_settings", {}).get("admin_users", []) + return user.get('email') in admin_users + +def require_user(request: Request): + user = get_current_user(request) + if not user: raise HTTPException(status_code=401, detail="Not authenticated") + return user + +def require_admin(request: Request): + if not is_admin(request): raise HTTPException(status_code=403, detail="Administrator privileges required.") + return True + +# --- CHUNKED UPLOADs --- +@app.post("/upload/chunk") +async def upload_chunk( + chunk: UploadFile = File(...), + upload_id: str = Form(...), + chunk_number: int = Form(...), + user: dict = Depends(require_user) # AUTHENTICATION +): + safe_upload_id = secure_filename(upload_id) + temp_dir = PATHS.CHUNK_TMP_DIR / safe_upload_id + temp_dir = ensure_path_is_safe(temp_dir, [PATHS.CHUNK_TMP_DIR]) + temp_dir.mkdir(exist_ok=True) + chunk_path = temp_dir / f"{chunk_number}.chunk" + + try: + with open(chunk_path, "wb") as buffer: + shutil.copyfileobj(chunk.file, buffer) + finally: + chunk.file.close() + return JSONResponse({"message": f"Chunk {chunk_number} for {safe_upload_id} uploaded."}) + +async def _stitch_chunks(temp_dir: Path, final_path: Path, total_chunks: int): + """Stitches chunks together and cleans up.""" + ensure_path_is_safe(temp_dir, [PATHS.CHUNK_TMP_DIR]) + ensure_path_is_safe(final_path, [PATHS.UPLOADS_DIR]) + with open(final_path, "wb") as final_file: + for i in range(total_chunks): + chunk_path = temp_dir / f"{i}.chunk" + if not chunk_path.exists(): + shutil.rmtree(temp_dir, ignore_errors=True) + raise HTTPException(status_code=400, detail=f"Upload failed: missing chunk {i}") + with open(chunk_path, "rb") as chunk_file: + final_file.write(chunk_file.read()) + shutil.rmtree(temp_dir, ignore_errors=True) + +@app.post("/upload/finalize", status_code=status.HTTP_202_ACCEPTED) +async def finalize_upload(payload: FinalizeUploadPayload, user: dict = Depends(require_user), db: Session = Depends(get_db)): + safe_upload_id = secure_filename(payload.upload_id) + temp_dir = PATHS.CHUNK_TMP_DIR / safe_upload_id + temp_dir = ensure_path_is_safe(temp_dir, [PATHS.CHUNK_TMP_DIR]) + if not temp_dir.is_dir(): + raise HTTPException(status_code=404, detail="Upload session not found or already finalized.") + + job_id = uuid.uuid4().hex + safe_filename = secure_filename(payload.original_filename) + final_path = PATHS.UPLOADS_DIR / f"{Path(safe_filename).stem}_{job_id}{Path(safe_filename).suffix}" + await _stitch_chunks(temp_dir, final_path, payload.total_chunks) + + job_data = JobCreate( + id=job_id, user_id=user['sub'], task_type=payload.task_type, + original_filename=payload.original_filename, input_filepath=str(final_path), + input_filesize=final_path.stat().st_size + ) + + if payload.task_type == "transcription": + stem = Path(safe_filename).stem + processed_path = PATHS.PROCESSED_DIR / f"{stem}_{job_id}.txt" + job_data.processed_filepath = str(processed_path) + create_job(db=db, job=job_data) + run_transcription_task(job_id, str(final_path), str(processed_path), payload.model_size, APP_CONFIG.get("transcription_settings", {}).get("whisper", {})) + elif payload.task_type == "ocr": + stem, suffix = Path(safe_filename).stem, Path(safe_filename).suffix + processed_path = PATHS.PROCESSED_DIR / f"{stem}_{job_id}{suffix}" + job_data.processed_filepath = str(processed_path) + create_job(db=db, job=job_data) + run_pdf_ocr_task(job_id, str(final_path), str(processed_path), APP_CONFIG.get("ocr_settings", {}).get("ocrmypdf", {})) + elif payload.task_type == "conversion": + try: + tool, task_key = payload.output_format.split('_', 1) + except Exception: + final_path.unlink(missing_ok=True) + raise HTTPException(status_code=400, detail="Invalid output_format for conversion.") + original_stem = Path(safe_filename).stem + target_ext = task_key.split('_')[0] + if tool == "ghostscript_pdf": target_ext = "pdf" + processed_path = PATHS.PROCESSED_DIR / f"{original_stem}_{job_id}.{target_ext}" + job_data.processed_filepath = str(processed_path) + create_job(db=db, job=job_data) + run_conversion_task(job_id, str(final_path), str(processed_path), tool, task_key, APP_CONFIG.get("conversion_tools", {})) + else: + final_path.unlink(missing_ok=True) + raise HTTPException(status_code=400, detail="Invalid task type.") + + return {"job_id": job_id, "status": "pending"} + +# --- OLD DIRECT-UPLOAD ROUTES (kept for compatibility) --- +# These use the same task functions but accept direct file uploads (no chunking). async def save_upload_file_chunked(upload_file: UploadFile, destination: Path) -> int: """ Write upload to a tmp file in chunks, then atomically move to final destination. Returns the final size of the file in bytes. """ max_size = APP_CONFIG.get("app_settings", {}).get("max_file_size_bytes", 100 * 1024 * 1024) - # make a temp filename that keeps the real extension, e.g. file.tmp-.pdf tmp = destination.with_name(f"{destination.stem}.tmp-{uuid.uuid4().hex}{destination.suffix}") size = 0 try: @@ -497,23 +749,27 @@ async def save_upload_file_chunked(upload_file: UploadFile, destination: Path) - tmp.replace(destination) return size except Exception: - tmp.unlink(missing_ok=True) + try: + ensure_path_is_safe(tmp, [PATHS.UPLOADS_DIR, PATHS.CHUNK_TMP_DIR]) + tmp.unlink(missing_ok=True) + except Exception: + logger.exception("Failed to remove temp upload file after error.") raise def is_allowed_file(filename: str, allowed_extensions: set) -> bool: return Path(filename).suffix.lower() in allowed_extensions -# --- Routes (transcription route uses Huey task enqueuing) --- - @app.post("/transcribe-audio", status_code=status.HTTP_202_ACCEPTED) async def submit_audio_transcription( file: UploadFile = File(...), model_size: str = Form("base"), - db: Session = Depends(get_db) + db: Session = Depends(get_db), + user: dict = Depends(require_user) ): - if not is_allowed_file(file.filename, {".mp3", ".wav", ".m4a", ".flac", ".ogg", ".opus"}): + allowed_audio_exts = {".mp3", ".wav", ".m4a", ".flac", ".ogg", ".opus"} + if not is_allowed_file(file.filename, allowed_audio_exts): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid audio file type.") - + whisper_config = APP_CONFIG.get("transcription_settings", {}).get("whisper", {}) if model_size not in whisper_config.get("allowed_models", []): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Invalid model size: {model_size}.") @@ -521,132 +777,41 @@ async def submit_audio_transcription( job_id = uuid.uuid4().hex safe_basename = secure_filename(file.filename) stem, suffix = Path(safe_basename).stem, Path(safe_basename).suffix - + audio_filename = f"{stem}_{job_id}{suffix}" transcript_filename = f"{stem}_{job_id}.txt" upload_path = PATHS.UPLOADS_DIR / audio_filename processed_path = PATHS.PROCESSED_DIR / transcript_filename input_size = await save_upload_file_chunked(file, upload_path) - + job_data = JobCreate( - id=job_id, - task_type="transcription", - original_filename=file.filename, - input_filepath=str(upload_path), + id=job_id, + user_id=user['sub'], + task_type="transcription", + original_filename=file.filename, + input_filepath=str(upload_path), input_filesize=input_size, processed_filepath=str(processed_path) ) new_job = create_job(db=db, job=job_data) - - # enqueue the Huey task (decorated function call enqueues when using huey) - run_transcription_task(new_job.id, str(upload_path), str(processed_path), model_size=model_size, whisper_settings=whisper_config) - + + run_transcription_task(new_job.id, str(upload_path), str(processed_path), model_size, whisper_settings=whisper_config) + return {"job_id": new_job.id, "status": new_job.status, "status_url": f"/job/{new_job.id}"} - -@app.get("/") -async def get_index(request: Request): - whisper_models = APP_CONFIG.get("transcription_settings", {}).get("whisper", {}).get("allowed_models", []) - conversion_tools = APP_CONFIG.get("conversion_tools", {}) - return templates.TemplateResponse("index.html", { - "request": request, - "whisper_models": sorted(list(whisper_models)), - "conversion_tools": conversion_tools - }) - -@app.get("/settings") -async def get_settings_page(request: Request): - try: - with open(PATHS.SETTINGS_FILE, 'r', encoding='utf8') as f: - current_config = yaml.safe_load(f) or {} - except Exception: - logger.exception("Could not load settings.yml for settings page") - current_config = {} - return templates.TemplateResponse("settings.html", {"request": request, "config": current_config}) - -def deep_merge(base: dict, updates: dict) -> dict: - """ - Recursively merge `updates` into `base`. Lists and scalars are replaced. - """ - for key, value in updates.items(): - if ( - key in base - and isinstance(base[key], dict) - and isinstance(value, dict) - ): - base[key] = deep_merge(base[key], value) - else: - base[key] = value - return base - - -@app.post("/settings/save") -async def save_settings(new_config: Dict = Body(...)): - tmp = PATHS.SETTINGS_FILE.with_suffix(".tmp") - try: - # load existing config if present - try: - with PATHS.SETTINGS_FILE.open("r", encoding="utf8") as f: - current_config = yaml.safe_load(f) or {} - except FileNotFoundError: - current_config = {} - - # deep merge new values - merged = deep_merge(current_config, new_config) - - # atomic write back - with tmp.open("w", encoding="utf8") as f: - yaml.safe_dump(merged, f, default_flow_style=False, sort_keys=False) - tmp.replace(PATHS.SETTINGS_FILE) - - load_app_config() - return JSONResponse({"message": "Settings updated successfully."}) - except Exception: - logger.exception("Failed to update settings") - tmp.unlink(missing_ok=True) - raise HTTPException(status_code=500, detail="Could not update settings.yml.") - - -@app.post("/settings/clear-history") -async def clear_job_history(db: Session = Depends(get_db)): - try: - num_deleted = db.query(Job).delete() - db.commit() - logger.info(f"Cleared {num_deleted} jobs from history.") - return {"deleted_count": num_deleted} - except Exception: - db.rollback() - logger.exception("Failed to clear job history") - raise HTTPException(status_code=500, detail="Database error while clearing history.") - -@app.post("/settings/delete-files") -async def delete_processed_files(): - deleted_count = 0 - errors = [] - for f in PATHS.PROCESSED_DIR.glob('*'): - try: - if f.is_file(): - f.unlink() - deleted_count += 1 - except Exception: - errors.append(f.name) - logger.exception(f"Could not delete processed file {f.name}") - if errors: - raise HTTPException(status_code=500, detail=f"Could not delete some files: {', '.join(errors)}") - logger.info(f"Deleted {deleted_count} files from processed directory.") - return {"deleted_count": deleted_count} - @app.post("/convert-file", status_code=status.HTTP_202_ACCEPTED) -async def submit_file_conversion(file: UploadFile = File(...), output_format: str = Form(...), db: Session = Depends(get_db)): +async def submit_file_conversion(file: UploadFile = File(...), output_format: str = Form(...), db: Session = Depends(get_db), user: dict = Depends(require_user)): allowed_exts = APP_CONFIG.get("app_settings", {}).get("allowed_all_extensions", set()) if not is_allowed_file(file.filename, allowed_exts): raise HTTPException(status_code=400, detail=f"File type '{Path(file.filename).suffix}' not allowed.") conversion_tools = APP_CONFIG.get("conversion_tools", {}) try: tool, task_key = output_format.split('_', 1) - if tool not in conversion_tools or task_key not in conversion_tools[tool]["formats"]: - raise ValueError() + if tool not in conversion_tools or task_key not in conversion_tools[tool].get("formats", {}): + # fallback: allow tasks that exist but may not be in formats map (some configs only have commands) + if tool not in conversion_tools: + raise ValueError() except ValueError: raise HTTPException(status_code=400, detail="Invalid output format selected.") job_id = uuid.uuid4().hex @@ -660,8 +825,8 @@ async def submit_file_conversion(file: UploadFile = File(...), output_format: st upload_path = PATHS.UPLOADS_DIR / upload_filename processed_path = PATHS.PROCESSED_DIR / processed_filename input_size = await save_upload_file_chunked(file, upload_path) - job_data = JobCreate(id=job_id, task_type="conversion", original_filename=file.filename, - input_filepath=str(upload_path), + job_data = JobCreate(id=job_id, user_id=user['sub'], task_type="conversion", original_filename=file.filename, + input_filepath=str(upload_path), input_filesize=input_size, processed_filepath=str(processed_path)) new_job = create_job(db=db, job=job_data) @@ -669,7 +834,7 @@ async def submit_file_conversion(file: UploadFile = File(...), output_format: st return {"job_id": new_job.id, "status": new_job.status, "status_url": f"/job/{new_job.id}"} @app.post("/ocr-pdf", status_code=status.HTTP_202_ACCEPTED) -async def submit_pdf_ocr(file: UploadFile = File(...), db: Session = Depends(get_db)): +async def submit_pdf_ocr(file: UploadFile = File(...), db: Session = Depends(get_db), user: dict = Depends(require_user)): if not is_allowed_file(file.filename, {".pdf"}): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid file type. Please upload a PDF.") job_id = uuid.uuid4().hex @@ -678,8 +843,8 @@ async def submit_pdf_ocr(file: UploadFile = File(...), db: Session = Depends(get upload_path = PATHS.UPLOADS_DIR / unique_filename processed_path = PATHS.PROCESSED_DIR / unique_filename input_size = await save_upload_file_chunked(file, upload_path) - job_data = JobCreate(id=job_id, task_type="ocr", original_filename=file.filename, - input_filepath=str(upload_path), + job_data = JobCreate(id=job_id, user_id=user['sub'], task_type="ocr", original_filename=file.filename, + input_filepath=str(upload_path), input_filesize=input_size, processed_filepath=str(processed_path)) new_job = create_job(db=db, job=job_data) @@ -688,7 +853,7 @@ async def submit_pdf_ocr(file: UploadFile = File(...), db: Session = Depends(get return {"job_id": new_job.id, "status": new_job.status, "status_url": f"/job/{new_job.id}"} @app.post("/ocr-image", status_code=status.HTTP_202_ACCEPTED) -async def submit_image_ocr(file: UploadFile = File(...), db: Session = Depends(get_db)): +async def submit_image_ocr(file: UploadFile = File(...), db: Session = Depends(get_db), user: dict = Depends(require_user)): allowed_exts = {".png", ".jpg", ".jpeg", ".tiff", ".tif"} if not is_allowed_file(file.filename, allowed_exts): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid file type. Please upload a PNG, JPG, or TIFF.") @@ -699,18 +864,242 @@ async def submit_image_ocr(file: UploadFile = File(...), db: Session = Depends(g upload_path = PATHS.UPLOADS_DIR / unique_filename processed_path = PATHS.PROCESSED_DIR / f"{Path(safe_basename).stem}_{job_id}.txt" input_size = await save_upload_file_chunked(file, upload_path) - job_data = JobCreate(id=job_id, task_type="ocr-image", original_filename=file.filename, - input_filepath=str(upload_path), + job_data = JobCreate(id=job_id, user_id=user['sub'], task_type="ocr-image", original_filename=file.filename, + input_filepath=str(upload_path), input_filesize=input_size, processed_filepath=str(processed_path)) new_job = create_job(db=db, job=job_data) run_image_ocr_task(new_job.id, str(upload_path), str(processed_path)) return {"job_id": new_job.id, "status": new_job.status, "status_url": f"/job/{new_job.id}"} +# --- Routes for auth and pages --- +if not LOCAL_ONLY_MODE: + @app.get('/login') + async def login(request: Request): + redirect_uri = request.url_for('auth') + return await oauth.oidc.authorize_redirect(request, redirect_uri) + + @app.get('/auth') + async def auth(request: Request): + try: + token = await oauth.oidc.authorize_access_token(request) + user = await oauth.oidc.userinfo(token=token) + request.session['user'] = dict(user) + # Store id_token in session for logout + request.session['id_token'] = token.get('id_token') + except Exception as e: + logger.error(f"Authentication failed: {e}") + raise HTTPException(status_code=401, detail="Authentication failed") + return RedirectResponse(url='/') + + @app.get("/logout") + async def logout(request: Request): + logout_endpoint = oauth.oidc.server_metadata.get("end_session_endpoint") + logger.info(f"OIDC end_session_endpoint: {logout_endpoint}") + + # local-only logout if provider doesn't expose end_session_endpoint + if not logout_endpoint: + request.session.clear() + logger.warning("OIDC 'end_session_endpoint' not found. Performing local-only logout.") + return RedirectResponse(url="/", status_code=302) + + + # Prefer a single canonical / registered post-logout redirect URI from config + post_logout_redirect_uri = str(request.url_for("get_index")) + logger.info(f"Post logout redirect URI: {post_logout_redirect_uri}") + + logout_url = f"{logout_endpoint}?post_logout_redirect_uri={post_logout_redirect_uri}" + + logger.info(f"Redirecting to provider logout URL: {logout_url}") + + request.session.clear() + return RedirectResponse(url=logout_url, status_code=302) + + +#### TODO: Remove this weird forward authz endpoint, its needed if reverse proxy does foward auth + +@app.get("/api/authz/forward-auth") +async def forward_auth(request: Request): + redirect_uri = request.url_for('auth') + return await oauth.oidc.authorize_redirect(request, redirect_uri) + +@app.get("/") +async def get_index(request: Request): + user = get_current_user(request) + admin_status = is_admin(request) + whisper_models = APP_CONFIG.get("transcription_settings", {}).get("whisper", {}).get("allowed_models", []) + conversion_tools = APP_CONFIG.get("conversion_tools", {}) + return templates.TemplateResponse("index.html", { + "request": request, + "user": user, + "is_admin": admin_status, + "whisper_models": sorted(list(whisper_models)), + "conversion_tools": conversion_tools, + "local_only_mode": LOCAL_ONLY_MODE + }) + +@app.get("/settings") +async def get_settings_page(request: Request): + """ + Displays the contents of the currently active configuration file. + It prioritizes settings.yml and falls back to settings.default.yml. + """ + user = get_current_user(request) + admin_status = is_admin(request) + current_config = {} + config_source = "none" # A helper variable to track which file was loaded + + try: + # 1. Attempt to load the primary, user-provided settings.yml + with open(PATHS.SETTINGS_FILE, 'r', encoding='utf8') as f: + current_config = yaml.safe_load(f) or {} + config_source = str(PATHS.SETTINGS_FILE.name) + logger.info(f"Displaying configuration from '{config_source}' on settings page.") + + except FileNotFoundError: + logger.warning(f"'{PATHS.SETTINGS_FILE.name}' not found. Attempting to display fallback configuration.") + try: + # 2. If it's not found, fall back to the default settings file + with open(PATHS.DEFAULT_SETTINGS_FILE, 'r', encoding='utf8') as f: + current_config = yaml.safe_load(f) or {} + config_source = str(PATHS.DEFAULT_SETTINGS_FILE.name) + logger.info(f"Displaying configuration from fallback '{config_source}' on settings page.") + + except Exception as e_fallback: + # 3. If even the default file fails, log the error and use an empty config + logger.exception(f"CRITICAL: Could not load fallback '{PATHS.DEFAULT_SETTINGS_FILE.name}' for settings page: {e_fallback}") + current_config = {} # Failsafe + config_source = "error" + + except Exception as e_primary: + # Handles other errors with the primary settings.yml (e.g., parsing errors, permissions) + logger.exception(f"Could not load '{PATHS.SETTINGS_FILE.name}' for settings page: {e_primary}") + current_config = {} # Failsafe + config_source = "error" + + return templates.TemplateResponse( + "settings.html", + { + "request": request, + "config": current_config, + "config_source": config_source, # You can use this in the template! + "user": user, + "is_admin": admin_status, + "local_only_mode": LOCAL_ONLY_MODE, + } + ) + + +import collections.abc + +def deep_merge(source: dict, destination: dict) -> dict: + """ + Recursively merges the `source` dictionary into the `destination` dictionary. + + Values from `source` will overwrite values in `destination`. + """ + for key, value in source.items(): + if isinstance(value, collections.abc.Mapping): + # If the value is a dictionary, recurse + node = destination.setdefault(key, {}) + deep_merge(value, node) + else: + # Otherwise, overwrite the value + destination[key] = value + return destination + +@app.post("/settings/save") +async def save_settings( + request: Request, + new_config_from_ui: Dict = Body(...), + admin: bool = Depends(require_admin) +): + """ + Safely updates settings.yml by merging UI changes with the existing file, + preserving any settings not managed by the UI. + """ + tmp_path = PATHS.SETTINGS_FILE.with_suffix(".tmp") + user = get_current_user(request) + + try: + # Handle the special case where the user wants to revert to defaults + if not new_config_from_ui: + if PATHS.SETTINGS_FILE.exists(): + PATHS.SETTINGS_FILE.unlink() + logger.info(f"Admin '{user.get('email')}' reverted to default settings by deleting settings.yml.") + load_app_config() + return JSONResponse({"message": "Settings reverted to default."}) + + # --- Read-Modify-Write Cycle --- + + # 1. READ: Load the current configuration from settings.yml on disk. + # If the file doesn't exist, start with an empty dictionary. + try: + with PATHS.SETTINGS_FILE.open("r", encoding="utf8") as f: + current_config_on_disk = yaml.safe_load(f) or {} + except FileNotFoundError: + current_config_on_disk = {} + + # 2. MODIFY: Deep merge the changes from the UI into the config from the disk. + # The UI config (`source`) overwrites keys in the disk config (`destination`). + merged_config = deep_merge(source=new_config_from_ui, destination=current_config_on_disk) + + # 3. WRITE: Save the fully merged configuration back to the file. + with tmp_path.open("w", encoding="utf8") as f: + yaml.safe_dump(merged_config, f, default_flow_style=False, sort_keys=False) + + tmp_path.replace(PATHS.SETTINGS_FILE) + logger.info(f"Admin '{user.get('email')}' successfully updated settings.yml.") + + # Reload the app config to apply changes immediately + load_app_config() + + return JSONResponse({"message": "Settings saved successfully. The new configuration is now active."}) + + except Exception as e: + logger.exception(f"Failed to update settings for admin '{user.get('email')}'") + if tmp_path.exists(): + tmp_path.unlink() + raise HTTPException(status_code=500, detail=f"Could not save settings.yml: {e}") + +# job management endpoints + +@app.post("/settings/clear-history") +async def clear_job_history(db: Session = Depends(get_db), user: dict = Depends(require_user)): + try: + num_deleted = db.query(Job).filter(Job.user_id == user['sub']).delete() + db.commit() + logger.info(f"Cleared {num_deleted} jobs from history for user {user['sub']}.") + return {"deleted_count": num_deleted} + except Exception: + db.rollback() + logger.exception("Failed to clear job history") + raise HTTPException(status_code=500, detail="Database error while clearing history.") + +@app.post("/settings/delete-files") +async def delete_processed_files(db: Session = Depends(get_db), user: dict = Depends(require_user)): + deleted_count = 0 + errors = [] + user_jobs = get_jobs(db, user_id=user['sub']) + for job in user_jobs: + if job.processed_filepath: + try: + p = ensure_path_is_safe(Path(job.processed_filepath), [PATHS.PROCESSED_DIR]) + if p.is_file(): + p.unlink() + deleted_count += 1 + except Exception as e: + errors.append(Path(job.processed_filepath).name) + logger.exception(f"Could not delete processed file {Path(job.processed_filepath).name}") + if errors: + raise HTTPException(status_code=500, detail=f"Could not delete some files: {', '.join(errors)}") + logger.info(f"Deleted {deleted_count} files from processed directory for user {user['sub']}.") + return {"deleted_count": deleted_count} + @app.post("/job/{job_id}/cancel", status_code=status.HTTP_202_ACCEPTED) -async def cancel_job(job_id: str, db: Session = Depends(get_db)): +async def cancel_job(job_id: str, db: Session = Depends(get_db), user: dict = Depends(require_user)): job = get_job(db, job_id) - if not job: + if not job or job.user_id != user['sub']: raise HTTPException(status_code=404, detail="Job not found.") if job.status in ["pending", "processing"]: update_job_status(db, job_id, status="cancelled") @@ -718,30 +1107,28 @@ async def cancel_job(job_id: str, db: Session = Depends(get_db)): raise HTTPException(status_code=400, detail=f"Job is already in a final state ({job.status}).") @app.get("/jobs", response_model=List[JobSchema]) -async def get_all_jobs(db: Session = Depends(get_db)): - return get_jobs(db) +async def get_all_jobs(db: Session = Depends(get_db), user: dict = Depends(require_user)): + return get_jobs(db, user_id=user['sub']) @app.get("/job/{job_id}", response_model=JobSchema) -async def get_job_status(job_id: str, db: Session = Depends(get_db)): +async def get_job_status(job_id: str, db: Session = Depends(get_db), user: dict = Depends(require_user)): job = get_job(db, job_id) - if not job: + if not job or job.user_id != user['sub']: raise HTTPException(status_code=404, detail="Job not found.") return job @app.get("/download/{filename}") -async def download_file(filename: str): +async def download_file(filename: str, db: Session = Depends(get_db), user: dict = Depends(require_user)): safe_filename = secure_filename(filename) - file_path = (PATHS.PROCESSED_DIR / safe_filename).resolve() - base = PATHS.PROCESSED_DIR.resolve() - try: - file_path.relative_to(base) - except ValueError: - raise HTTPException(status_code=403, detail="Access denied.") + file_path = ensure_path_is_safe(PATHS.PROCESSED_DIR / safe_filename, [PATHS.PROCESSED_DIR]) if not file_path.is_file(): raise HTTPException(status_code=404, detail="File not found.") - return FileResponse(path=file_path, filename=safe_filename, media_type="application/octet-stream") + job = db.query(Job).filter(Job.processed_filepath == str(file_path), Job.user_id == user['sub']).first() + if not job: + raise HTTPException(status_code=403, detail="You do not have permission to download this file.") + download_filename = Path(job.original_filename).stem + Path(job.processed_filepath).suffix + return FileResponse(path=file_path, filename=download_filename, media_type="application/octet-stream") -# Small health endpoint @app.get("/health") async def health(): try: @@ -752,8 +1139,6 @@ async def health(): return JSONResponse({"ok": False}, status_code=500) return {"ok": True} -favicon_path = PATHS.BASE_DIR / 'static' / 'favicon.png' - @app.get('/favicon.ico', include_in_schema=False) async def favicon(): - return FileResponse(favicon_path) \ No newline at end of file + return FileResponse(str(PATHS.BASE_DIR / 'static' / 'favicon.png')) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8e07724..91db382 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ pypdf # Configuration & Utilities werkzeug +dotenv PyYAML pydantic pydantic-settings diff --git a/run.sh b/run.sh index 15d2a67..0e28b9e 100755 --- a/run.sh +++ b/run.sh @@ -2,9 +2,14 @@ # Exit immediately if a command exits with a non-zero status. set -e +source .env +LOCAL_ONLY=True +SECRET_KEY=4bd5dcfc32602307e5101efcfedbb7c3e770a8167d596933 +UPLOADS_DIR=/home/manuel/file-wiz/uploads +PROCESSED_DIR=/home/manuel/file-wiz/processed # Start Gunicorn in the background -gunicorn -w 4 --threads 2 -k uvicorn.workers.UvicornWorker --forwarded-allow-ips='*' main:app -b 0.0.0.0:8000 & +gunicorn -w 4 --threads 2 -k uvicorn.workers.UvicornWorker --forwarded-allow-ips='*' --error-logfile - --access-logfile - main:app -b 0.0.0.0:8000 & echo "Started Gunicorn..." # Store the Gunicorn process ID GUNICORN_PID=$! diff --git a/settings .yml.default b/settings.default.yml similarity index 87% rename from settings .yml.default rename to settings.default.yml index 8c17157..327907b 100644 --- a/settings .yml.default +++ b/settings.default.yml @@ -1,45 +1,14 @@ +auth_settings: + oidc_client_id: + oidc_client_secret: + oidc_server_metadata_url: https://accounts.oauth.com/oidc/.well-known/openid-configuration + oidc_userinfo_endpoint: https://accounts.oauth.com/oidc/me + oidc_end_session_endpoint: https://accounts.oauth.com/oidc/session/end + admin_users: + - admin@local.com app_settings: max_file_size_mb: '2000' - allowed_all_extensions: - - .pdf - - .ps - - .eps - - .png - - .jpg - - .jpeg - - .tiff - - .tif - - .gif - - .bmp - - .webp - - .svg - - .jxl - - .avif - - .ppm - - .mp3 - - .m4a - - .ogg - - .flac - - .opus - - .wav - - .aac - - .mp4 - - .mkv - - .mov - - .webm - - .avi - - .flv - - .md - - .txt - - .html - - .docx - - .odt - - .rst - - .epub - - .mobi - - .azw3 - - .pptx - - .xlsx + allowed_all_extensions: [.pdf, .ps, .eps, .png, .jpg, .jpeg, .tiff, .tif, .gif, .bmp, .webp, .svg, .jxl, .avif, .ppm, .mp3, .m4a, .ogg, .flac, .opus, .wav, .aac, .mp4, .mkv, .mov, .webm, .avi, .flv, .md, .txt, .html, .docx, .odt, .rst, .epub, .mobi, .azw3, .pptx, .xlsx] ocr_settings: ocrmypdf: deskew: true @@ -223,15 +192,15 @@ conversion_tools: png_fast: PNG (Fast Compression) sox: name: SoX Audio Converter - command_template: sox {input} -r {samplerate} -b {bitdepth} {output} + command_template: sox {input} -r {samplerate} {bitdepth} {output} timeout: 600 formats: wav_48k_24b: WAV (48kHz, 24-bit) wav_44k_16b: WAV (CD, 44.1kHz, 16-bit) flac_48k_24b: FLAC (48kHz, 24-bit) flac_44k_16b: FLAC (CD, 44.1kHz, 16-bit) - ogg_32k_16b: Ogg Vorbis (32kHz) - ogg_16k_16b: Ogg Vorbis (16kHz, Voice) + ogg_32k: Ogg Vorbis (32kHz) + ogg_16k: Ogg Vorbis (16kHz, Voice) mozjpeg: name: MozJPEG command_template: cjpeg -quality {quality} -outfile {output} {input} diff --git a/settings.yml b/settings.yml index 8c17157..f132fe3 100644 --- a/settings.yml +++ b/settings.yml @@ -1,45 +1,80 @@ +auth_settings: + oidc_client_id: filewiz + oidc_client_secret: 5tIR4k75pdmiV2QSlLSSeqsA5vgxB21F + oidc_server_metadata_url: https://accounts.manuelunterriker.de/oidc/.well-known/openid-configuration + oidc_userinfo_endpoint: https://accounts.manuelunterriker.de/oidc/me + oidc_end_session_endpoint: https://accounts.manuelunterriker.de/oidc/session/end + admin_users: + - manuel@unterriker.com app_settings: max_file_size_mb: '2000' allowed_all_extensions: - - .pdf - - .ps - - .eps - - .png - - .jpg - - .jpeg - - .tiff - - .tif - - .gif - - .bmp - - .webp - - .svg - - .jxl - - .avif - - .ppm - - .mp3 - - .m4a - - .ogg - - .flac - - .opus - - .wav - .aac - - .mp4 - - .mkv - - .mov - - .webm + - .aiff + - .avif - .avi - - .flv - - .md - - .txt - - .html - - .docx - - .odt - - .rst - - .epub - - .mobi - .azw3 + - .bmp + - .csv + - .dbf + - .doc + - .docx + - .dpx + - .dxf + - .eps + - .epub + - .fb2 + - .fits + - .flac + - .flv + - .gif + - .gsm + - .html + - .htmlz + - .ico + - .jpeg + - .jpg + - .jxl + - .lit + - .lrf + - .m4a + - .md + - .mkv + - .mobi + - .mov + - .mp3 + - .mp4 + - .odt + - .ogg + - .ogv + - .opml + - .opus + - .pdb + - .pdf + - .pnm + - .png + - .ppt - .pptx + - .ps + - .rst + - .rtf + - .svg + - .tcr + - .tex + - .tif + - .tiff + - .txt + - .wav + - .webm + - .webp + - .wma + - .wmv + - .wps - .xlsx + - .xls + - .xml + - .xpm + - .zip ocr_settings: ocrmypdf: deskew: true @@ -65,6 +100,7 @@ conversion_tools: filters: pdf: pdf docx: docx + doc: doc odt: odt html: html rtf: rtf @@ -72,26 +108,39 @@ conversion_tools: xml: xml epub: epub xlsx: xlsx + xls: xls ods: ods csv: csv:Text pptx: pptx + ppt: ppt odp: odp svg: svg + png: png + jpg: jpg + wps: wps + dbf: dbf formats: - pdf: PDF - docx: Word Document + pdf: PDF Document + docx: Word Document (DOCX) + doc: Word 97-2003 Document (DOC) odt: OpenDocument Text - html: HTML + html: HTML Document rtf: Rich Text Format txt: Plain Text xml: Word 2003 XML - epub: EPUB - xlsx: Excel Spreadsheet + epub: EPUB E-Book + xlsx: Excel Spreadsheet (XLSX) + xls: Excel 97-2003 Spreadsheet (XLS) ods: OpenDocument Spreadsheet csv: CSV - pptx: PowerPoint Presentation + pptx: PowerPoint Presentation (PPTX) + ppt: PowerPoint 97-2003 Presentation (PPT) odp: OpenDocument Presentation - svg: SVG + svg: SVG Image (from Draw/Impress) + png: PNG Image (from Draw/Impress) + jpg: JPEG Image (from Draw/Impress) + wps: MS Works Document + dbf: dBase Database File pandoc: name: Pandoc command_template: pandoc --standalone {input} -o {output} --to={output_ext} --pdf-engine=xelatex @@ -99,22 +148,27 @@ conversion_tools: formats: docx: Word Document odt: OpenDocument Text - pdf: PDF + pdf: PDF Document rtf: Rich Text Format txt: Plain Text + md: Markdown (Strict) + gfm: Markdown (GitHub-Flavored) tex: LaTeX - man: Groff Man Page + html: HTML5 epub: EPUB v3 Book epub2: EPUB v2 Book - html: HTML - html5: HTML5 pptx: PowerPoint Presentation beamer: Beamer PDF Slides slidy: Slidy HTML Slides - md: Markdown + revealjs: reveal.js HTML Slides rst: reStructuredText jira: Jira Wiki Markup mediawiki: MediaWiki Markup + asciidoc: AsciiDoc + textile: Textile + opml: OPML + man: Groff Man Page + docbook: DocBook XML ghostscript_pdf: name: Ghostscript (PDF) command_template: gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dNOPAUSE -dQUIET @@ -124,7 +178,8 @@ conversion_tools: screen: PDF (Optimized for Screen) ebook: PDF (Optimized for Ebooks) printer: PDF (Optimized for Print) - archive: PDF/A (for Archiving) + prepress: PDF (Optimized for Prepress) + pdfa: PDF/A-2b (for Archiving) ghostscript_image: name: Ghostscript (Image) command_template: gs -dNOPAUSE -dBATCH -sDEVICE={device} -r{dpi} -sOutputFile={output} @@ -132,11 +187,18 @@ conversion_tools: timeout: 60 formats: jpeg_72: JPEG Image (72 DPI) + jpeg_150: JPEG Image (150 DPI) jpeg_300: JPEG Image (300 DPI) png16m_150: PNG Image (150 DPI) png16m_300: PNG Image (300 DPI) + pngalpha_150: PNG Image with Alpha (150 DPI) + pngalpha_300: PNG Image with Alpha (300 DPI) tiff24nc_300: TIFF Image (300 DPI) tiff24nc_600: TIFF Image (600 DPI) + tiffg4_300: TIFF Image (G4 Fax, 300 DPI) + bmp16m_300: BMP Image (300 DPI) + pcx24b_300: PCX Image (300 DPI) + pnm_300: PNM Image (300 DPI) calibre: name: Calibre (ebook-convert) command_template: ebook-convert {input} {output} @@ -144,22 +206,42 @@ conversion_tools: formats: epub: EPUB mobi: MOBI - azw3: Amazon Kindle + azw3: Amazon Kindle (AZW3) pdf: PDF docx: Word Document + rtf: Rich Text Format + txt: Plain Text + fb2: FictionBook 2 + htmlz: Zipped HTML + pdb: eReader PDB + lrf: Sony BroadBand eBook + lit: Microsoft Reader + tcr: Psion Series 3 ffmpeg: name: FFmpeg command_template: ffmpeg -i {input} -y -preset medium {output} timeout: 600 formats: - mp4: MP4 Video - mkv: MKV Video - mov: MOV Video - webm: WebM Video + mp4: MP4 Video (H.264/AAC) + mp4_hevc: MP4 Video (H.265/AAC) + mkv: MKV Video (H.264/AAC) + mov: MOV Video (H.264/AAC) + webm: WebM Video (VP9/Opus) + webm_av1: WebM Video (AV1/Opus) + avi: AVI Video (MPEG4/MP3) + wmv: WMV Video + flv: FLV Video (Flash) + ogv: Ogg Theora Video mp3: MP3 Audio - wav: WAV Audio - flac: FLAC Audio + wav: WAV Audio (Uncompressed PCM) + flac: FLAC Audio (Lossless) + aac: AAC Audio + aiff: AIFF Audio + wma: WMA Audio + ogg: Ogg Vorbis Audio + opus: Opus Audio gif: Animated GIF + apng: Animated PNG vips: name: VIPS command_template: vips copy {input} {output}[Q=90] @@ -169,7 +251,12 @@ conversion_tools: png: PNG Image webp: WebP Image (Q90) tiff: TIFF Image - avif: AVIF Image + avif: AVIF Image (Q90) + heif: HEIF Image (Q90) + jp2: JPEG 2000 + gif: GIF Image + pnm: PNM Image + fits: FITS Image graphicsmagick: name: GraphicsMagick command_template: gm convert {input} -quality 90 {output} @@ -179,21 +266,36 @@ conversion_tools: png: PNG Image webp: WebP Image tiff: TIFF Image + gif: GIF Image + bmp: BMP Image pdf: PDF from Images + eps: Encapsulated PostScript + dpx: DPX Image + ico: Windows Icon + xpm: X PixMap inkscape: name: Inkscape command_template: inkscape {input} --export-filename={output} timeout: 30 formats: svg: SVG (Plain) - png: PNG Image (96 DPI) + inkscape_svg: SVG (Inkscape) + png_96: PNG Image (96 DPI) + png_300: PNG Image (300 DPI) pdf: PDF Document + eps: Encapsulated PostScript + ps: PostScript + emf: Enhanced Metafile + wmf: Windows Metafile + dxf: AutoCAD DXF R14 libjxl: name: libjxl (cjxl) command_template: cjxl {input} {output} -q 90 timeout: 30 formats: - jxl: JPEG XL (Q90) + jxl_q90: JPEG XL (Quality 90) + jxl_lossless: JPEG XL (Lossless) + jxl_hq: JPEG XL (High Compression) resvg: name: resvg command_template: resvg {input} {output} @@ -206,6 +308,10 @@ conversion_tools: timeout: 30 formats: svg: SVG from Bitmap + pdf: PDF from Bitmap + eps: EPS from Bitmap + ps: PostScript from Bitmap + dxf: DXF from Bitmap markitdown: name: Markitdown command_template: markitdown {input} -o {output} @@ -223,15 +329,22 @@ conversion_tools: png_fast: PNG (Fast Compression) sox: name: SoX Audio Converter - command_template: sox {input} -r {samplerate} -b {bitdepth} {output} + command_template: sox {input} -r {samplerate} {bitdepth} {output} timeout: 600 formats: wav_48k_24b: WAV (48kHz, 24-bit) wav_44k_16b: WAV (CD, 44.1kHz, 16-bit) + wav_16k_16b: WAV (Voice, 16kHz, 16-bit) flac_48k_24b: FLAC (48kHz, 24-bit) flac_44k_16b: FLAC (CD, 44.1kHz, 16-bit) - ogg_32k_16b: Ogg Vorbis (32kHz) - ogg_16k_16b: Ogg Vorbis (16kHz, Voice) + aiff_44k_16b: AIFF (CD, 44.1kHz, 16-bit) + mp3_320k: MP3 (320 kbps) + mp3_128k: MP3 (128 kbps) + ogg_192k: Ogg Vorbis (192 kbps) + ogg_96k: Ogg Vorbis (96 kbps) + opus_128k: Opus (128 kbps) + opus_64k: Opus (64 kbps, Voice) + gsm: GSM 06.10 (13kbps, Voice) mozjpeg: name: MozJPEG command_template: cjpeg -quality {quality} -outfile {output} {input} diff --git a/static/css/style.css b/static/css/style.css index 69a3381..9ae80aa 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -39,10 +39,61 @@ body { -moz-osx-font-smoothing: grayscale; } +/* --- START: Login View Styles --- */ +#login-container { + display: flex; /* Initially hidden/shown by JS */ + flex-direction: column; + justify-content: center; + align-items: center; + height: 100vh; + text-align: center; + padding: 1rem; +} + +.login-box { + background: var(--card-bg); + padding: 2.5rem 3rem; + border-radius: 10px; + border: 1px solid var(--border-color); +} + +#login-container h1 { + font-family: serif; + font-weight: lighter; + font-size: 3rem; + margin: 0 0 0.5rem 0; +} + +#login-container p { + font-size: 1.1rem; + color: var(--muted-text); + margin-bottom: 2rem; +} + +.login-button-style { + background-color: var(--primary-color); + color: var(--bg-color); + border: 1px solid var(--primary-color); + padding: 0.7rem 2.5rem; + font-size: 1rem; + font-weight: 600; + border-radius: 5px; + cursor: pointer; + transition: background-color 0.15s ease, color 0.15s ease, transform 0.1s ease; +} + +.login-button-style:hover { + background-color: transparent; + color: var(--primary-color); + transform: scale(1.03); +} +/* --- END: Login View Styles --- */ + + /* Container */ .container { width: 100%; - max-width: 1280px; /* Increased max-width for 3 columns */ + max-width: 1280px; margin: 0 auto; background: var(--card-bg); border-radius: 10px; @@ -71,7 +122,6 @@ header { header h1 { margin: 0 0 0.25rem 0; font-size: 3rem; - font-weight: 700; font-family: serif; font-weight: lighter; } @@ -86,34 +136,50 @@ header p { position: absolute; top: 1.5rem; right: 1.5rem; + display: flex; + align-items: center; + gap: 1rem; +} + +.header-actions div { + position: fixed; + left: 0; + bottom: 0em; + width: 100vw; + padding: 2em; + padding-top: 5em; + text-align: center; + background-color: rgb(0, 0, 0); + background: linear-gradient(0deg, rgb(0, 0, 0) 30%, rgba(0, 0, 0, 0) 100%); +} + +.user-info { + font-size: 0.9rem; + color: var(--muted-text); } .settings-link { - font-size: 1.5rem; + font-size: 0.9rem; + margin-left: 0.5em; text-decoration: none; color: var(--muted-text); transition: color 0.2s ease; + } .settings-link:hover { color: var(--text-color); } -/* Form Layout */ -.form-grid { - display: grid; - /* MODIFICATION: Responsive grid for 1, 2, or 3 items */ - grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); - gap: 1.5rem; + +/* --- Form and Input Styles --- */ +.processor-section { margin-bottom: 2rem; } .upload-form fieldset { border: 1px solid var(--border-color); border-radius: 8px; - padding: 1rem; + padding: 1.5rem; margin: 0; background: transparent; - height: 100%; - display: flex; - flex-direction: column; } .upload-form legend { @@ -134,7 +200,6 @@ header p { align-items: center; gap: 1rem; margin-top: 1rem; - margin-bottom: 1rem; } input[type="file"] { @@ -211,31 +276,56 @@ input[type="file"] { box-shadow: 0 0 0 2px rgba(0, 180, 255, 0.2); } -/* Submit button */ -button[type="submit"] { +/* --- NEW: Responsive group for action sections --- */ +.actions-group { + display: grid; + grid-template-columns: repeat(3, 1fr); /* 3 columns for wide screens */ + gap: 1.5rem; + margin-top: 1.5rem; +} + +.action-fieldset { + /* Inherits border/padding from the generic fieldset style */ + display: flex; + flex-direction: column; +} + +.action-description { + color: var(--muted-text); + font-size: 0.9rem; + margin-bottom: 1rem; + flex-grow: 1; /* Helps align buttons if one section has more text */ +} + +/* --- NEW: Styling for individual action buttons --- */ +.main-action-button { display: block; width: 100%; - background: var(--primary-color); - background-color: transparent; - border-color: var(--border-color); - border-width: 1px; - color: #ffffff; + background: transparent; + border: 1px solid var(--border-color); + color: var(--text-color); padding: 0.65rem 1rem; font-size: 1rem; font-weight: 600; border-radius: 5px; cursor: pointer; - transition: background-color 0.15s ease; - margin-top: auto; + transition: background-color 0.15s ease, border-color 0.15s ease; + margin-top: auto; /* Pushes button to the bottom of the flex container (fieldset) */ } -button[type="submit"]:hover { + +.main-action-button:hover { background: var(--primary-hover); + border-color: var(--primary-hover); } -button[type="submit"]:disabled { + +.main-action-button:disabled { background-color: var(--muted-text); + border-color: var(--muted-text); cursor: not-allowed; + opacity: 0.5; } + /* History Section */ #job-history h2 { text-align: center; @@ -251,12 +341,14 @@ button[type="submit"]:disabled { border: 1px solid var(--border-color); border-radius: 8px; background-color: var(--surface); + margin-bottom: 6em; } #job-table { width: 100%; border-collapse: collapse; font-size: 0.9rem; + margin-bottom: 2em; } #job-table th, @@ -390,7 +482,7 @@ button[type="submit"]:disabled { } .cancel-button:hover { background-color: #ff8f8f; } -/* --- MODIFICATION: Dark theme for Choices.js --- */ +/* --- Dark theme for Choices.js --- */ .choices { font-size: 0.95rem; } @@ -447,7 +539,7 @@ button[type="submit"]:disabled { } -/* --- START: Drag and Drop and Dialog Styles --- */ +/* --- Drag and Drop and Dialog Styles --- */ .drag-overlay { position: fixed; inset: 0; @@ -549,7 +641,6 @@ body.dragging .drag-overlay { .dialog-cancel:hover { color: var(--text-color); } -/* --- END: Drag and Drop and Dialog Styles --- */ /* Spinner */ .spinner-small { @@ -567,7 +658,13 @@ body.dragging .drag-overlay { 100% { transform: rotate(360deg); } } -/* Mobile responsive table */ +/* --- Mobile responsive styles --- */ +@media (max-width: 992px) { + .actions-group { + grid-template-columns: 1fr; /* Stack to 1 column on tablet and smaller */ + } +} + @media (max-width: 768px) { .table-wrapper { border: none; diff --git a/static/js/script.js b/static/js/script.js index c9b54da..7d23215 100644 --- a/static/js/script.js +++ b/static/js/script.js @@ -1,6 +1,9 @@ document.addEventListener('DOMContentLoaded', () => { - // --- User Locale and Timezone Detection (Corrected Implementation) --- - const USER_LOCALE = navigator.language || 'en-US'; // Fallback to en-US + // --- Constants --- + const CHUNK_SIZE = 5 * 1024 * 1024; // 5 MB chunks + + // --- User Locale and Timezone Detection --- + const USER_LOCALE = navigator.language || 'en-US'; const USER_TIMEZONE = Intl.DateTimeFormat().resolvedOptions().timeZone; const DATETIME_FORMAT_OPTIONS = { year: 'numeric', @@ -10,67 +13,69 @@ document.addEventListener('DOMContentLoaded', () => { minute: '2-digit', timeZone: USER_TIMEZONE, }; - console.log(`Using locale: ${USER_LOCALE} and timezone: ${USER_TIMEZONE}`); // --- Element Selectors --- - const jobListBody = document.getElementById('job-list-body'); - - const pdfForm = document.getElementById('pdf-form'); - const pdfFileInput = document.getElementById('pdf-file-input'); - const pdfFileName = document.getElementById('pdf-file-name'); - - const audioForm = document.getElementById('audio-form'); - const audioFileInput = document.getElementById('audio-file-input'); - const audioFileName = document.getElementById('audio-file-name'); - const modelSizeSelect = document.getElementById('model-size-select'); - - const conversionForm = document.getElementById('conversion-form'); - const conversionFileInput = document.getElementById('conversion-file-input'); - const conversionFileName = document.getElementById('conversion-file-name'); - const outputFormatSelect = document.getElementById('output-format-select'); + const appContainer = document.getElementById('app-container'); + const loginContainer = document.getElementById('login-container'); + const loginButton = document.getElementById('login-button'); - // START: Drag and Drop additions + // Main form elements + const mainFileInput = document.getElementById('main-file-input'); + const mainFileName = document.getElementById('main-file-name'); + const mainOutputFormatSelect = document.getElementById('main-output-format-select'); + const mainModelSizeSelect = document.getElementById('main-model-size-select'); + const startConversionBtn = document.getElementById('start-conversion-btn'); + const startOcrBtn = document.getElementById('start-ocr-btn'); + const startTranscriptionBtn = document.getElementById('start-transcription-btn'); + + const jobListBody = document.getElementById('job-list-body'); + + // Drag and Drop Elements const dragOverlay = document.getElementById('drag-overlay'); const actionDialog = document.getElementById('action-dialog'); const dialogFileCount = document.getElementById('dialog-file-count'); - // Dialog Views const dialogInitialView = document.getElementById('dialog-initial-actions'); const dialogConvertView = document.getElementById('dialog-convert-view'); - // Dialog Buttons const dialogConvertBtn = document.getElementById('dialog-action-convert'); const dialogOcrBtn = document.getElementById('dialog-action-ocr'); const dialogTranscribeBtn = document.getElementById('dialog-action-transcribe'); const dialogCancelBtn = document.getElementById('dialog-action-cancel'); const dialogStartConversionBtn = document.getElementById('dialog-start-conversion'); const dialogBackBtn = document.getElementById('dialog-back'); - // Dialog Select const dialogOutputFormatSelect = document.getElementById('dialog-output-format-select'); - // END: Drag and Drop additions + // --- State Variables --- let conversionChoices = null; - let dialogConversionChoices = null; // For the dialog's format selector + let modelChoices = null; // For the model dropdown instance + let dialogConversionChoices = null; const activePolls = new Map(); - let stagedFiles = null; // To hold files from a drop event + let stagedFiles = null; - // --- Main Event Listeners --- - pdfFileInput.addEventListener('change', () => updateFileName(pdfFileInput, pdfFileName)); - audioFileInput.addEventListener('change', () => updateFileName(audioFileInput, audioFileName)); - conversionFileInput.addEventListener('change', () => updateFileName(conversionFileInput, conversionFileName)); - pdfForm.addEventListener('submit', (e) => handleFormSubmit(e, '/ocr-pdf', pdfForm)); - audioForm.addEventListener('submit', (e) => handleFormSubmit(e, '/transcribe-audio', audioForm)); - conversionForm.addEventListener('submit', (e) => handleFormSubmit(e, '/convert-file', conversionForm)); - - jobListBody.addEventListener('click', (event) => { - if (event.target.classList.contains('cancel-button')) { - const jobId = event.target.dataset.jobId; - handleCancelJob(jobId); + // --- Authentication-aware Fetch Wrapper --- + /** + * A wrapper around the native fetch API that handles 401 Unauthorized responses. + * If a 401 is received, it assumes the session has expired and redirects to the login page. + * @param {string} url - The URL to fetch. + * @param {object} options - The options for the fetch request. + * @returns {Promise} - A promise that resolves to the fetch Response. + */ + async function authFetch(url, options) { + const response = await fetch(url, options); + if (response.status === 401) { + // Use a simple alert for now. A more sophisticated modal could be used. + alert('Your session has expired. You will be redirected to the login page.'); + window.location.href = '/login'; + // Throw an error to stop the promise chain of the calling function + throw new Error('Session expired'); } - }); + return response; + } + // --- Helper Functions --- function formatBytes(bytes, decimals = 1) { - if (!+bytes) return '0 Bytes'; // Handles 0, null, undefined + if (!+bytes) return '0 Bytes'; const k = 1024; const dm = decimals < 0 ? 0 : decimals; const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; @@ -78,109 +83,166 @@ document.addEventListener('DOMContentLoaded', () => { return `${parseFloat((bytes / Math.pow(k, i)).toFixed(dm))} ${sizes[i]}`; } - // --- Core Job Submission Logic (Refactored for reuse) --- - async function submitJob(endpoint, formData, originalFilename) { - try { - const response = await fetch(endpoint, { method: 'POST', body: formData }); - if (!response.ok) { - const errorData = await response.json(); - throw new Error(errorData.detail || `HTTP error! Status: ${response.status}`); + // --- Chunked Uploading Logic --- + async function uploadFileInChunks(file, taskType, options = {}) { + const uploadId = 'upload-' + Date.now() + '-' + Math.random().toString(36).substr(2, 9); + const totalChunks = Math.ceil(file.size / CHUNK_SIZE); + + const preliminaryJob = { + id: uploadId, + status: 'uploading', + progress: 0, + original_filename: file.name, + input_filesize: file.size, + task_type: taskType, + created_at: new Date().toISOString() + }; + renderJobRow(preliminaryJob); + + for (let chunkNumber = 0; chunkNumber < totalChunks; chunkNumber++) { + const start = chunkNumber * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, file.size); + const chunk = file.slice(start, end); + const formData = new FormData(); + formData.append('chunk', chunk, file.name); + formData.append('upload_id', uploadId); + formData.append('chunk_number', chunkNumber); + + try { + const response = await authFetch('/upload/chunk', { + method: 'POST', + body: formData, + }); + if (!response.ok) { + throw new Error(`Chunk upload failed with status: ${response.status}`); + } + const progress = Math.round(((chunkNumber + 1) / totalChunks) * 100); + updateUploadProgress(uploadId, progress); + } catch (error) { + console.error(`Error uploading chunk ${chunkNumber} for ${file.name}:`, error); + if (error.message !== 'Session expired') { + updateJobToFailedState(uploadId, `Upload failed: ${error.message}`); + } + return; } - const result = await response.json(); - const preliminaryJob = { - id: result.job_id, - status: 'pending', - progress: 0, - original_filename: originalFilename, - input_filesize: formData.get('file').size, - task_type: endpoint.includes('ocr') ? 'ocr' : (endpoint.includes('transcribe') ? 'transcription' : 'conversion'), - created_at: new Date().toISOString() // Create preliminary UTC timestamp + } + + try { + const finalizePayload = { + upload_id: uploadId, + original_filename: file.name, + total_chunks: totalChunks, + task_type: taskType, + ...options }; - renderJobRow(preliminaryJob); + const finalizeResponse = await authFetch('/upload/finalize', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(finalizePayload), + }); + if (!finalizeResponse.ok) { + const errorData = await finalizeResponse.json(); + throw new Error(errorData.detail || 'Finalization failed'); + } + const result = await finalizeResponse.json(); + const uploadRow = document.getElementById(uploadId); + if (uploadRow) { + uploadRow.id = `job-${result.job_id}`; + const statusCell = uploadRow.querySelector('td[data-label="Status"] .cell-value'); + if (statusCell) { + statusCell.innerHTML = `Pending`; + } + } startPolling(result.job_id); } catch (error) { - console.error('Error submitting job:', error); - alert(`Submission failed for ${originalFilename}: ${error.message}`); + console.error(`Error finalizing upload for ${file.name}:`, error); + if (error.message !== 'Session expired') { + updateJobToFailedState(uploadId, `Finalization failed: ${error.message}`); + } } } - // --- Original Form Submission Handler (Now uses submitJob) --- - async function handleFormSubmit(event, endpoint, form) { - event.preventDefault(); - const fileInput = form.querySelector('input[type="file"]'); - if (fileInput.files.length === 0) return; - - const submitButton = form.querySelector('button[type="submit"]'); - submitButton.disabled = true; - - // Convert FileList to an array to loop through it - const files = Array.from(fileInput.files); - - // Process each file as a separate job - for (const file of files) { - const formData = new FormData(); - formData.append('file', file); - - // Append other form data if it exists - const outputFormat = form.querySelector('select[name="output_format"]'); - if (outputFormat) { - formData.append('output_format', outputFormat.value); + function updateUploadProgress(uploadId, progress) { + const row = document.getElementById(uploadId); + if (row) { + const progressBar = row.querySelector('.progress-bar'); + if (progressBar) { + progressBar.style.width = `${progress}%`; } - const modelSize = form.querySelector('select[name="model_size"]'); - if (modelSize) { - formData.append('model_size', modelSize.value); - } - - // Await each job submission to process them sequentially - await submitJob(endpoint, formData, file.name); } - - // Reset the form UI after all jobs have been submitted - const fileNameDisplay = form.querySelector('.file-name'); - form.reset(); - if (fileNameDisplay) { - fileNameDisplay.textContent = 'No file chosen'; - fileNameDisplay.title = 'No file chosen'; - } - if (form.id === 'conversion-form' && conversionChoices) { - conversionChoices.clearInput(); - conversionChoices.setValue([]); - } - submitButton.disabled = false; } - // --- START: Drag and Drop Implementation --- - function setupDragAndDropListeners() { - let dragCounter = 0; // Counter to manage enter/leave events reliably + function updateJobToFailedState(jobId, errorMessage) { + const row = document.getElementById(jobId); + if (row) { + const statusCell = row.querySelector('td[data-label="Status"] .cell-value'); + const actionCell = row.querySelector('td[data-label="Action"] .cell-value'); + if (statusCell) statusCell.innerHTML = `Failed`; + if (actionCell) { + const errorTitle = errorMessage ? ` title="${errorMessage.replace(/"/g, '"')}"` : ''; + actionCell.innerHTML = `Failed`; + } + } + } + // --- Centralized Task Request Handler --- + async function handleTaskRequest(taskType) { + if (mainFileInput.files.length === 0) { + alert('Please choose one or more files first.'); + return; + } + + const files = Array.from(mainFileInput.files); + const options = {}; + + if (taskType === 'conversion') { + const selectedFormat = conversionChoices.getValue(true); + if (!selectedFormat) { + alert('Please select a format to convert to.'); + return; + } + options.output_format = selectedFormat; + } else if (taskType === 'transcription') { + options.model_size = mainModelSizeSelect.value; + } + + // Disable buttons during upload process + startConversionBtn.disabled = true; + startOcrBtn.disabled = true; + startTranscriptionBtn.disabled = true; + + const uploadPromises = files.map(file => uploadFileInChunks(file, taskType, options)); + await Promise.allSettled(uploadPromises); + + // Reset file input and re-enable buttons + mainFileInput.value = ''; // Resets the file list + updateFileName(mainFileInput, mainFileName); + startConversionBtn.disabled = false; + startOcrBtn.disabled = false; + startTranscriptionBtn.disabled = false; + } + + + function setupDragAndDropListeners() { + let dragCounter = 0; window.addEventListener('dragenter', (e) => { e.preventDefault(); dragCounter++; document.body.classList.add('dragging'); }); - window.addEventListener('dragleave', (e) => { e.preventDefault(); dragCounter--; - if (dragCounter === 0) { - document.body.classList.remove('dragging'); - } + if (dragCounter === 0) document.body.classList.remove('dragging'); }); - - window.addEventListener('dragover', (e) => { - e.preventDefault(); // This is necessary to allow a drop - }); - + window.addEventListener('dragover', (e) => e.preventDefault()); window.addEventListener('drop', (e) => { e.preventDefault(); - dragCounter = 0; // Reset counter + dragCounter = 0; document.body.classList.remove('dragging'); - - // Only handle the drop if it's on our designated overlay if (e.target === dragOverlay || dragOverlay.contains(e.target)) { - const files = e.dataTransfer.files; - if (files && files.length > 0) { - stagedFiles = files; + if (e.dataTransfer.files && e.dataTransfer.files.length > 0) { + stagedFiles = e.dataTransfer.files; showActionDialog(); } } @@ -189,16 +251,8 @@ document.addEventListener('DOMContentLoaded', () => { function showActionDialog() { dialogFileCount.textContent = stagedFiles.length; - - // Clone options from main form's select to the dialog's select - dialogOutputFormatSelect.innerHTML = outputFormatSelect.innerHTML; - - // Clean up previous Choices.js instance if it exists - if (dialogConversionChoices) { - dialogConversionChoices.destroy(); - } - - // Initialize a new Choices.js instance for the dialog + dialogOutputFormatSelect.innerHTML = mainOutputFormatSelect.innerHTML; // Use main select as template + if (dialogConversionChoices) dialogConversionChoices.destroy(); dialogConversionChoices = new Choices(dialogOutputFormatSelect, { searchEnabled: true, itemSelectText: 'Select', @@ -206,8 +260,6 @@ document.addEventListener('DOMContentLoaded', () => { placeholder: true, placeholderValue: 'Select a format...', }); - - // Ensure the initial view is shown dialogInitialView.style.display = 'grid'; dialogConvertView.style.display = 'none'; actionDialog.classList.add('visible'); @@ -216,91 +268,61 @@ document.addEventListener('DOMContentLoaded', () => { function closeActionDialog() { actionDialog.classList.remove('visible'); stagedFiles = null; - // Important: Destroy the Choices instance to prevent memory leaks if (dialogConversionChoices) { - // Explicitly hide the dropdown before destroying - dialogConversionChoices.hideDropdown(); + dialogConversionChoices.hideDropdown(); dialogConversionChoices.destroy(); dialogConversionChoices = null; } } - // --- Dialog Button and Action Listeners --- dialogConvertBtn.addEventListener('click', () => { - // Switch to the conversion view dialogInitialView.style.display = 'none'; dialogConvertView.style.display = 'block'; }); - dialogBackBtn.addEventListener('click', () => { - // Switch back to the initial view dialogInitialView.style.display = 'grid'; dialogConvertView.style.display = 'none'; }); - - dialogStartConversionBtn.addEventListener('click', () => handleDialogAction('convert')); + dialogStartConversionBtn.addEventListener('click', () => handleDialogAction('conversion')); dialogOcrBtn.addEventListener('click', () => handleDialogAction('ocr')); - dialogTranscribeBtn.addEventListener('click', () => handleDialogAction('transcribe')); + dialogTranscribeBtn.addEventListener('click', () => handleDialogAction('transcription')); dialogCancelBtn.addEventListener('click', closeActionDialog); - function handleDialogAction(action) { if (!stagedFiles) return; - - let endpoint = ''; - const formDataArray = []; - - for (const file of stagedFiles) { - const formData = new FormData(); - formData.append('file', file); - - if (action === 'convert') { - const selectedFormat = dialogConversionChoices.getValue(true); - if (!selectedFormat) { - alert('Please select a format to convert to.'); - return; - } - formData.append('output_format', selectedFormat); - endpoint = '/convert-file'; - } else if (action === 'ocr') { - endpoint = '/ocr-pdf'; - } else if (action === 'transcribe') { - formData.append('model_size', modelSizeSelect.value); - endpoint = '/transcribe-audio'; + let options = {}; + if (action === 'conversion') { + const selectedFormat = dialogConversionChoices.getValue(true); + if (!selectedFormat) { + alert('Please select a format to convert to.'); + return; } - formDataArray.push({ formData, name: file.name }); + options.output_format = selectedFormat; + } else if (action === 'transcription') { + options.model_size = mainModelSizeSelect.value; } - - formDataArray.forEach(item => { - submitJob(endpoint, item.formData, item.name); - }); - + Array.from(stagedFiles).forEach(file => uploadFileInChunks(file, action, options)); closeActionDialog(); } - // --- END: Drag and Drop Implementation --- - function initializeConversionSelector() { - if (conversionChoices) { - conversionChoices.destroy(); - } - conversionChoices = new Choices(outputFormatSelect, { + /** + * Initializes all Choices.js dropdowns on the page. + */ + function initializeSelectors() { + // --- Conversion Dropdown --- + if (conversionChoices) conversionChoices.destroy(); + conversionChoices = new Choices(mainOutputFormatSelect, { searchEnabled: true, itemSelectText: 'Select', shouldSort: false, placeholder: true, placeholderValue: 'Select a format...', }); - const tools = window.APP_CONFIG.conversionTools || {}; const choicesArray = []; for (const toolKey in tools) { const tool = tools[toolKey]; - const group = { - label: tool.name, - id: toolKey, - disabled: false, - choices: [] - }; + const group = { label: tool.name, id: toolKey, disabled: false, choices: [] }; for (const formatKey in tool.formats) { group.choices.push({ value: `${toolKey}_${formatKey}`, @@ -310,20 +332,23 @@ document.addEventListener('DOMContentLoaded', () => { choicesArray.push(group); } conversionChoices.setChoices(choicesArray, 'value', 'label', true); + + // --- Model Size Dropdown --- + if (modelChoices) modelChoices.destroy(); + modelChoices = new Choices(mainModelSizeSelect, { + searchEnabled: false, // Disables the search box + shouldSort: false, // Keeps the original