diff --git a/.dockerignore b/.dockerignore old mode 100644 new mode 100755 diff --git a/.env.example b/.env.example old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/Dockerfile b/Dockerfile old mode 100644 new mode 100755 index b6fc9fd..d10b485 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ pandoc texlive-xetex \ texlive-latex-recommended \ texlive-fonts-recommended \ + unpaper \ calibre \ ffmpeg \ libvips-tools \ diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/docker-compose.yml b/docker-compose.yml old mode 100644 new mode 100755 diff --git a/main.py b/main.py old mode 100644 new mode 100755 index d617f91..a00ffd8 --- a/main.py +++ b/main.py @@ -30,11 +30,13 @@ import time import ocrmypdf import pypdf import pytesseract +from fastapi.middleware.cors import CORSMiddleware from pytesseract import TesseractNotFoundError from PIL import Image, UnidentifiedImageError from faster_whisper import WhisperModel from fastapi import (Depends, FastAPI, File, Form, HTTPException, Request, UploadFile, status, Body) +from fastapi.concurrency import run_in_threadpool from fastapi.responses import FileResponse, JSONResponse, RedirectResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates @@ -130,9 +132,20 @@ def _limit_resources_preexec(): logging.getLogger(__name__).warning(f"Could not set resource limits: {e}") pass -# --- Model concurrency semaphore --- -MODEL_CONCURRENCY = int(os.environ.get("MODEL_CONCURRENCY", "1")) -_model_semaphore = Semaphore(MODEL_CONCURRENCY) +# --- Model concurrency semaphore (lazily initialized) --- +_model_semaphore: Optional[Semaphore] = None + +def get_model_semaphore() -> Semaphore: + """Lazily initializes and returns the global model semaphore.""" + global _model_semaphore + if _model_semaphore is None: + # Read from app config, fall back to env var, then to a hardcoded default of 1 + model_concurrency_from_env = int(os.environ.get("MODEL_CONCURRENCY", "1")) + model_concurrency = APP_CONFIG.get("app_settings", {}).get("model_concurrency", model_concurrency_from_env) + _model_semaphore = Semaphore(model_concurrency) + logger.info(f"Model concurrency semaphore initialized with limit: {model_concurrency}") + return _model_semaphore + # --- Logging Setup --- logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') @@ -176,6 +189,7 @@ def load_app_config(): """ Loads configuration from settings.yml, with a fallback to settings.default.yml, and finally to hardcoded defaults if both files are missing. + Reads environment variables for transcription device settings. """ global APP_CONFIG try: @@ -183,9 +197,33 @@ def load_app_config(): with open(PATHS.SETTINGS_FILE, 'r', encoding='utf8') as f: cfg_raw = yaml.safe_load(f) or {} + # Read transcription settings from environment variables, providing smart defaults. + transcription_device = os.environ.get("TRANSCRIPTION_DEVICE", "cpu") + # Default to float16 for CUDA for better performance, otherwise int8 for CPU. + default_compute_type = "float16" if transcription_device == "cuda" else "int8" + transcription_compute_type = os.environ.get("TRANSCRIPTION_COMPUTE_TYPE", default_compute_type) + transcription_device_index_str = os.environ.get("TRANSCRIPTION_DEVICE_INDEX", "0") + + # Handle multiple device indexes (e.g., "0,1") + try: + if ',' in transcription_device_index_str: + transcription_device_index = [int(i.strip()) for i in transcription_device_index_str.split(',')] + else: + transcription_device_index = int(transcription_device_index_str) + except ValueError: + logger.warning(f"Invalid TRANSCRIPTION_DEVICE_INDEX value: '{transcription_device_index_str}'. Defaulting to 0.") + transcription_device_index = 0 + defaults = { "app_settings": {"max_file_size_mb": 100, "allowed_all_extensions": [], "app_public_url": ""}, - "transcription_settings": {"whisper": {"allowed_models": ["tiny", "base", "small"], "compute_type": "int8", "device": "cpu"}}, + "transcription_settings": { + "whisper": { + "allowed_models": ["tiny", "base", "small", "medium", "large-v1", "large-v2", "large-v3"], + "compute_type": transcription_compute_type, + "device": transcription_device, + "device_index": transcription_device_index + } + }, "tts_settings": { "piper": {"model_dir": str(PATHS.TTS_MODELS_DIR), "use_cuda": False, "synthesis_config": {"length_scale": 1.0, "noise_scale": 0.667, "noise_w": 0.8}}, "kokoro": {"model_dir": str(PATHS.KOKORO_TTS_MODELS_DIR), "command_template": "kokoro-tts {input} {output} --model {model_path} --voices {voices_path} --lang {lang} --voice {model_name}"} @@ -369,6 +407,8 @@ def create_job(db: Session, job: JobCreate): db.add(db_job) db.commit() db.refresh(db_job) + # Broadcast the new job to UI clients via Huey task + job_schema = JobSchema.model_validate(db_job) return db_job def update_job_status(db: Session, job_id: str, status: str, progress: int = None, error: str = None): @@ -381,6 +421,8 @@ def update_job_status(db: Session, job_id: str, status: str, progress: int = Non db_job.error_message = error db.commit() db.refresh(db_job) + # Broadcast the updated job to UI clients via Huey task + job_schema = JobSchema.model_validate(db_job) return db_job def mark_job_as_completed(db: Session, job_id: str, output_filepath_str: str | None = None, preview: str | None = None): @@ -398,6 +440,9 @@ def mark_job_as_completed(db: Session, job_id: str, output_filepath_str: str | N except Exception: logger.exception(f"Could not stat output file {output_filepath_str} for job {job_id}") db.commit() + db.refresh(db_job) + # Broadcast the final job state to UI clients via Huey task + job_schema = JobSchema.model_validate(db_job) return db_job @@ -461,45 +506,99 @@ huey = SqliteHuey(filename=PATHS.HUEY_DB_PATH) WHISPER_MODELS_CACHE: Dict[str, WhisperModel] = {} PIPER_VOICES_CACHE: Dict[str, "PiperVoice"] = {} AVAILABLE_TTS_VOICES_CACHE: Dict[str, Any] | None = None +WHISPER_MODELS_LAST_USED: Dict[str, float] = {} - +# --- Cache Eviction Settings --- +_cache_cleanup_thread: Optional[threading.Thread] = None +_cache_lock = threading.Lock() # Global lock for modifying cache dictionaries _model_locks: Dict[str, threading.Lock] = {} -_global_lock = threading.Lock() +_global_lock = threading.Lock() # Lock for initializing model-specific locks + +def _whisper_cache_cleanup_worker(): + """ + Periodically checks for and unloads Whisper models that have been inactive. + The timeout and check interval are configured in the application settings. + """ + while True: + # Read settings within the loop to allow for live changes + app_settings = APP_CONFIG.get("app_settings", {}) + check_interval = app_settings.get("cache_check_interval", 300) + inactivity_timeout = app_settings.get("model_inactivity_timeout", 1800) + + time.sleep(check_interval) + + with _cache_lock: + # Create a copy of items to avoid issues with modifying dict while iterating + expired_models = [] + for model_size, last_used in WHISPER_MODELS_LAST_USED.items(): + if time.time() - last_used > inactivity_timeout: + expired_models.append(model_size) + + if not expired_models: + continue + + logger.info(f"Found {len(expired_models)} inactive Whisper models to unload: {expired_models}") + + for model_size in expired_models: + # Acquire the specific model lock before removing to prevent race conditions + model_lock = _get_or_create_model_lock(model_size) + with model_lock: + # Check if the model is still in the cache (it should be) + if model_size in WHISPER_MODELS_CACHE: + logger.info(f"Unloading inactive Whisper model: {model_size}") + # Remove from caches + model_to_unload = WHISPER_MODELS_CACHE.pop(model_size, None) + WHISPER_MODELS_LAST_USED.pop(model_size, None) + + # Explicitly delete the object to encourage garbage collection + if model_to_unload: + del model_to_unload + + # Explicitly run garbage collection outside the main lock + import gc + gc.collect() def get_whisper_model(model_size: str, whisper_settings: dict) -> Any: - # Fast path: cache hit without any locking - if model_size in WHISPER_MODELS_CACHE: - logger.debug(f"Cache hit for model '{model_size}'") - return WHISPER_MODELS_CACHE[model_size] - - # Prepare for potential load with minimal contention - model_lock = _get_or_create_model_lock(model_size) - - # Critical section: check cache again under model-specific lock - with model_lock: + # Fast path: check cache. If hit, update timestamp and return. + with _cache_lock: if model_size in WHISPER_MODELS_CACHE: + logger.debug(f"Cache hit for model '{model_size}'") + WHISPER_MODELS_LAST_USED[model_size] = time.time() return WHISPER_MODELS_CACHE[model_size] - + + # Model not in cache, prepare for loading. + model_lock = _get_or_create_model_lock(model_size) + + with model_lock: + # Re-check cache inside lock in case another thread loaded it + with _cache_lock: + if model_size in WHISPER_MODELS_CACHE: + WHISPER_MODELS_LAST_USED[model_size] = time.time() + return WHISPER_MODELS_CACHE[model_size] + logger.info(f"Loading Whisper model '{model_size}'...") try: - # Optimized initialization with validated settings device = whisper_settings.get("device", "cpu") compute_type = whisper_settings.get("compute_type", "int8") - - # fast_whisper-specific optimizations + device_index = whisper_settings.get("device_index", 0) + model = WhisperModel( model_size, device=device, + device_index=device_index, compute_type=compute_type, - cpu_threads=max(1, os.cpu_count() // 2), # Prevent CPU oversubscription - num_workers=1 # Optimal for most transcription workloads + cpu_threads=max(1, os.cpu_count() // 2), + num_workers=1 ) - - # Atomic cache update - WHISPER_MODELS_CACHE[model_size] = model + + # Add the new model to the cache under lock + with _cache_lock: + WHISPER_MODELS_CACHE[model_size] = model + WHISPER_MODELS_LAST_USED[model_size] = time.time() + logger.info(f"Model '{model_size}' loaded (device={device}, compute={compute_type})") return model - + except Exception as e: logger.error(f"Model '{model_size}' failed to load: {str(e)}", exc_info=True) raise RuntimeError(f"Whisper model initialization failed: {e}") from e @@ -509,7 +608,7 @@ def _get_or_create_model_lock(model_size: str) -> threading.Lock: # Fast path: lock already exists if model_size in _model_locks: return _model_locks[model_size] - + # Slow path: create lock under global lock with _global_lock: return _model_locks.setdefault(model_size, threading.Lock()) @@ -543,7 +642,7 @@ def get_piper_voice(model_name: str, tts_settings: dict | None) -> "PiperVoice": logger.info("Reusing cached Piper voice '%s'.", model_name) return PIPER_VOICES_CACHE[model_name] - with _model_semaphore: + with get_model_semaphore(): if model_name in PIPER_VOICES_CACHE: return PIPER_VOICES_CACHE[model_name] @@ -963,178 +1062,46 @@ def list_kokoro_languages_cli(timeout: int = 60) -> List[str]: def run_command( argv: List[str], - timeout: int = 300, - max_output_size: int = 5 * 1024 * 1024 # 5MB + timeout: int = 300 ) -> subprocess.CompletedProcess: """ - Drop-in replacement for your run_command. - - Incrementally reads stdout/stderr in separate threads to avoid unbounded memory growth. - - Keeps at most `max_output_size` characters per stream (first N chars). - - Enforces a timeout (graceful terminate then kill). - - Uses optional preexec function `_limit_resources_preexec` if present in globals. - - Raises Exception on non-zero exit or timeout; returns CompletedProcess on success. + Executes a command, captures its output, and handles timeouts and errors. + Uses resource limits for child processes. This is a simplified, more robust + implementation using subprocess.run. """ - logger.debug("Executing command: %s with timeout=%ss", " ".join(argv), timeout) - - # quick sanity: ensure there's a program to execute (improves error clarity) - try: - exe = argv[0] - except Exception: - raise Exception("Invalid argv passed to run_command") + logger.debug("Executing command: %s with timeout=%ss", " ".join(shlex.quote(s) for s in argv), timeout) preexec = globals().get("_limit_resources_preexec", None) - # Buffers and state for threads - stdout_chunks = [] - stderr_chunks = [] - stdout_len = 0 - stderr_len = 0 - stdout_lock = threading.Lock() - stderr_lock = threading.Lock() - stdout_truncated = False - stderr_truncated = False - - def _reader(stream, chunks, lock, name): - nonlocal stdout_len, stderr_len, stdout_truncated, stderr_truncated - try: - while True: - data = stream.read(4096) - if not data: - break - with lock: - # choose which counters to use by stream identity - if name == "stdout": - if stdout_len < max_output_size: - # append as much as fits - remaining = max_output_size - stdout_len - to_append = data[:remaining] - chunks.append(to_append) - stdout_len += len(to_append) - if len(data) > remaining: - stdout_truncated = True - else: - stdout_truncated = True - else: - if stderr_len < max_output_size: - remaining = max_output_size - stderr_len - to_append = data[:remaining] - chunks.append(to_append) - stderr_len += len(to_append) - if len(data) > remaining: - stderr_truncated = True - else: - stderr_truncated = True - except Exception: - logger.exception("Reader thread for %s failed", name) - finally: - try: - stream.close() - except Exception: - pass - - # Start process try: - proc = subprocess.Popen( + # subprocess.run handles timeout, output capturing, and error checking. + result = subprocess.run( argv, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, text=True, + timeout=timeout, + check=True, # Raises CalledProcessError on non-zero exit preexec_fn=preexec ) + logger.debug("Command completed successfully: %s", " ".join(shlex.quote(s) for s in argv)) + return result except FileNotFoundError: msg = f"Command not found: {argv[0]}" logger.error(msg) - raise Exception(msg) + raise Exception(msg) from None + except subprocess.TimeoutExpired as e: + msg = f"Command timed out after {timeout}s: {' '.join(shlex.quote(s) for s in argv)}" + logger.error(msg) + raise Exception(msg) from e + except subprocess.CalledProcessError as e: + snippet = (e.stderr or "")[:1000] + msg = f"Command failed with exit code {e.returncode}. Stderr: {snippet}" + logger.error(msg) + raise Exception(msg) from e except Exception as e: msg = f"Unexpected error launching command: {e}" logger.exception(msg) - raise Exception(msg) - - # Start reader threads - t_stdout = threading.Thread(target=_reader, args=(proc.stdout, stdout_chunks, stdout_lock, "stdout"), daemon=True) - t_stderr = threading.Thread(target=_reader, args=(proc.stderr, stderr_chunks, stderr_lock, "stderr"), daemon=True) - t_stdout.start() - t_stderr.start() - - # Wait loop with timeout - start = time.monotonic() - try: - while True: - ret = proc.poll() - if ret is not None: - break - elapsed = time.monotonic() - start - if timeout and elapsed > timeout: - # Timeout -> try terminate -> kill if needed - logger.error("Command timed out after %ss: %s", timeout, " ".join(argv)) - try: - proc.terminate() - # give it a short while to exit - waited = 0.0 - while proc.poll() is None and waited < 2.0: - time.sleep(0.1) - waited += 0.1 - except Exception: - logger.exception("Failed to terminate process on timeout; attempting kill.") - if proc.poll() is None: - try: - proc.kill() - except Exception: - logger.exception("Failed to kill process after timeout.") - # ensure threads finish reading leftover data - try: - t_stdout.join(timeout=1.0) - t_stderr.join(timeout=1.0) - except Exception: - pass - raise Exception(f"Command timed out after {timeout}s: {' '.join(argv)}") - - # sleep a little to avoid busy loop - time.sleep(0.1) - - # process finished normally; allow readers to finish - t_stdout.join(timeout=2.0) - t_stderr.join(timeout=2.0) - - # build strings from chunks - with stdout_lock: - stdout_str = "".join(stdout_chunks) if stdout_chunks else "" - if stdout_truncated: - truncated_amount = " (truncated to max_output_size)" - stdout_str += f"\n[TRUNCATED - output larger than {max_output_size} bytes]{truncated_amount}" - with stderr_lock: - stderr_str = "".join(stderr_chunks) if stderr_chunks else "" - if stderr_truncated: - truncated_amount = " (truncated to max_output_size)" - stderr_str += f"\n[TRUNCATED - output larger than {max_output_size} bytes]{truncated_amount}" - - # Check return code - rc = proc.returncode - if rc != 0: - # include limited stderr snippet for diagnostics (like your original) - snippet = (stderr_str or "")[:1000] - msg = f"Command failed with exit code {rc}. Stderr: {snippet}" - logger.error(msg) - raise Exception(msg) - - logger.debug("Command completed successfully: %s", " ".join(argv)) - return subprocess.CompletedProcess(args=argv, returncode=rc, stdout=stdout_str, stderr=stderr_str) - - finally: - # ensure no resource leaks - try: - if proc.stdout: - try: - proc.stdout.close() - except Exception: - pass - if proc.stderr: - try: - proc.stderr.close() - except Exception: - pass - except Exception: - pass + raise Exception(msg) from e def validate_and_build_command(template_str: str, mapping: Dict[str, str]) -> TypingList[str]: fmt = Formatter() @@ -1147,12 +1114,21 @@ def validate_and_build_command(template_str: str, mapping: Dict[str, str]) -> Ty bad = used - ALLOWED_VARS if bad: raise ValueError(f"Command template contains disallowed placeholders: {bad}") + safe_mapping = dict(mapping) for name in used: if name not in safe_mapping: safe_mapping[name] = safe_mapping.get("output_ext", "") if name == "filter" else "" - formatted = template_str.format(**safe_mapping) - return shlex.split(formatted) + + # Securely build the command by splitting the template BEFORE formatting. + # This prevents argument injection if a value in the mapping (e.g. a filename) + # contains spaces or other shell-special characters. + command_parts = shlex.split(template_str) + + formatted_command = [part.format(**safe_mapping) for part in command_parts] + + # Filter out any empty strings that result from empty optional placeholders + return [part for part in formatted_command if part] # --- TASK RUNNERS --- @huey.task() @@ -1182,9 +1158,9 @@ def run_transcription_task(job_id: str, input_path_str: str, output_path_str: st segments_generator, info = model.transcribe(str(input_path), beam_size=5) logger.info(f"Detected language: {info.language} with probability {info.language_probability:.2f} for a duration of {info.duration:.2f}s") - + last_update_time = time.time() - + # Use a temporary file to ensure atomic writes. The final file will only appear # once the transcription is fully and successfully written. tmp_output_path = output_path.with_name(f"{output_path.stem}.tmp-{uuid.uuid4().hex}{output_path.suffix}") @@ -1204,7 +1180,7 @@ def run_transcription_task(job_id: str, input_path_str: str, output_path_str: st preview_segments.append(segment_text) current_preview_length += len(segment_text) - + current_time = time.time() if current_time - last_update_time > DB_POLL_INTERVAL_SECONDS: last_update_time = current_time @@ -1222,7 +1198,7 @@ def run_transcription_task(job_id: str, input_path_str: str, output_path_str: st update_job_status(db, job_id, "processing", progress=progress) tmp_output_path.replace(output_path) - + transcript_preview = " ".join(preview_segments) if len(transcript_preview) > PREVIEW_MAX_LENGTH: transcript_preview = transcript_preview[:PREVIEW_MAX_LENGTH] + "..." @@ -1237,7 +1213,7 @@ def run_transcription_task(job_id: str, input_path_str: str, output_path_str: st finally: # This block executes whether the task succeeded, failed, or was cancelled and returned. logger.debug(f"Performing cleanup for job {job_id}") - + # Clean up the temporary file if it still exists (e.g., due to cancellation) if 'tmp_output_path' in locals() and tmp_output_path.exists(): try: @@ -1254,10 +1230,19 @@ def run_transcription_task(job_id: str, input_path_str: str, output_path_str: st logger.debug(f"Removed input file: {input_path}") except Exception as e: logger.exception(f"Failed to cleanup input file {input_path} for job {job_id}: {e}") - + if db: db.close() - + + # If this was a sub-job, trigger a progress update for its parent. + db_for_check = SessionLocal() + try: + job = get_job(db_for_check, job_id) + if job and job.parent_job_id: + _update_parent_zip_job_progress(job.parent_job_id) + finally: + db_for_check.close() + # Send notification last, after all state has been finalized. send_webhook_notification(job_id, app_config, base_url) @@ -1344,6 +1329,16 @@ def run_tts_task(job_id: str, input_path_str: str, output_path_str: str, model_n except Exception: logger.exception("Failed to cleanup input file after TTS.") db.close() + + # If this was a sub-job, trigger a progress update for its parent. + db_for_check = SessionLocal() + try: + job = get_job(db_for_check, job_id) + if job and job.parent_job_id: + _update_parent_zip_job_progress(job.parent_job_id) + finally: + db_for_check.close() + send_webhook_notification(job_id, app_config, base_url) @huey.task() @@ -1377,6 +1372,16 @@ def run_pdf_ocr_task(job_id: str, input_path_str: str, output_path_str: str, ocr except Exception: logger.exception("Failed to cleanup input file after PDF OCR.") db.close() + + # If this was a sub-job, trigger a progress update for its parent. + db_for_check = SessionLocal() + try: + job = get_job(db_for_check, job_id) + if job and job.parent_job_id: + _update_parent_zip_job_progress(job.parent_job_id) + finally: + db_for_check.close() + send_webhook_notification(job_id, app_config, base_url) @huey.task() @@ -1493,6 +1498,16 @@ def run_image_ocr_task(job_id: str, input_path_str: str, output_path_str: str, a db.close() except Exception: logger.exception("Failed to close DB session after Image OCR.") + + # If this was a sub-job, trigger a progress update for its parent. + db_for_check = SessionLocal() + try: + job = get_job(db_for_check, job_id) + if job and job.parent_job_id: + _update_parent_zip_job_progress(job.parent_job_id) + finally: + db_for_check.close() + # send webhook regardless of success/failure (keeps original behavior) try: send_webhook_notification(job_id, app_config, base_url) @@ -1756,6 +1771,15 @@ def run_conversion_task(job_id: str, except Exception: logger.exception("Failed to close DB session after conversion.") + # If this was a sub-job, trigger a progress update for its parent. + db_for_check = SessionLocal() + try: + job = get_job(db_for_check, job_id) + if job and job.parent_job_id: + _update_parent_zip_job_progress(job.parent_job_id) + finally: + db_for_check.close() + try: gc.collect() except Exception: @@ -1820,27 +1844,214 @@ def dispatch_single_file_job(original_filename: str, input_filepath: str, task_t run_pdf_ocr_task(job_data.id, str(final_path), str(processed_path), app_config.get("ocr_settings", {}).get("ocrmypdf", {}), app_config, base_url) elif task_type == "conversion": try: - tool, task_key = options.get("output_format").split('_', 1) + logger.info(f"Preparing to dispatch conversion job for file '{original_filename}' with requested format '{options.get('output_format')}'") + all_tools = app_config.get("conversion_tools", {}).keys() + logger.info(f"Available conversion tools: {', '.join(all_tools)}") + tool, task_key = _parse_tool_and_task_key(options.get("output_format"), all_tools) + logger.info(f"Dispatching conversion job using tool '{tool}' with task key '{task_key}' for file '{original_filename}'") except (AttributeError, ValueError): - logger.error(f"Invalid or missing output_format for conversion of {original_filename}") - final_path.unlink(missing_ok=True) - return + if parent_job_id: + logger.warning(f"Skipping file '{original_filename}' from batch job '{parent_job_id}' as it is not applicable for the selected conversion format '{options.get('output_format')}'.") + final_path.unlink(missing_ok=True) + return + else: + logger.error(f"Invalid or missing output_format for conversion of {original_filename}") + final_path.unlink(missing_ok=True) + return + original_stem = Path(safe_filename).stem - target_ext = task_key.split('_')[0] - if tool == "ghostscript_pdf": target_ext = "pdf" - processed_path = PATHS.PROCESSED_DIR / f"{original_stem}_{job_id}.{target_ext}" - job_data.processed_filepath = str(processed_path) - create_job(db=db, job=job_data) - run_conversion_task(job_data.id, str(final_path), str(processed_path), tool, task_key, app_config.get("conversion_tools", {}), app_config, base_url) + + if tool == 'pandoc_academic': + processed_path = PATHS.PROCESSED_DIR / f"{original_stem}_{job_id}.pdf" + job_data.processed_filepath = str(processed_path) + job_data.task_type = 'academic_pandoc' # Use a more specific task type for the DB + create_job(db=db, job=job_data) + run_academic_pandoc_task(job_data.id, str(final_path), str(processed_path), task_key, APP_CONFIG, base_url) + else: + target_ext = task_key.split('_')[0] + if tool == "ghostscript_pdf": target_ext = "pdf" + processed_path = PATHS.PROCESSED_DIR / f"{original_stem}_{job_id}.{target_ext}" + job_data.processed_filepath = str(processed_path) + create_job(db=db, job=job_data) + run_conversion_task(job_data.id, str(final_path), str(processed_path), tool, task_key, app_config.get("conversion_tools", {}), app_config, base_url) else: logger.error(f"Invalid task type '{task_type}' for file {original_filename}") final_path.unlink(missing_ok=True) +@huey.task() +def run_academic_pandoc_task(job_id: str, input_path_str: str, output_path_str: str, task_key: str, app_config: dict, base_url: str): + """ + Runs a Pandoc conversion for a zipped academic project (e.g., markdown + bibliography). + """ + db = SessionLocal() + input_path = Path(input_path_str) + output_path = Path(output_path_str) + unzip_dir = PATHS.UPLOADS_DIR / f"unzipped_{job_id}" + + def find_first_file_with_ext(directory: Path, extensions: List[str]) -> Optional[Path]: + for ext in extensions: + try: + return next(directory.rglob(f"*{ext}")) + except StopIteration: + continue + return None + + try: + job = get_job(db, job_id) + if not job or job.status == 'cancelled': + return + + update_job_status(db, job_id, "processing", progress=10) + logger.info(f"Starting academic Pandoc task for job {job_id}") + + # 1. Unzip the project + if not zipfile.is_zipfile(input_path): + raise ValueError("Input is not a valid ZIP archive.") + unzip_dir.mkdir() + with zipfile.ZipFile(input_path, 'r') as zip_ref: + zip_ref.extractall(unzip_dir) + + update_job_status(db, job_id, "processing", progress=25) + + # 2. Find required files + main_doc = find_first_file_with_ext(unzip_dir, ['.md', '.tex', '.txt']) + bib_file = find_first_file_with_ext(unzip_dir, ['.bib']) + csl_file = find_first_file_with_ext(unzip_dir, ['.csl']) + + if not main_doc: + raise FileNotFoundError("No main document (.md, .tex, .txt) found in the ZIP archive.") + if not bib_file: + raise FileNotFoundError("No bibliography file (.bib) found in the ZIP archive.") + + update_job_status(db, job_id, "processing", progress=40) + + # 3. Build Pandoc command + command = ['pandoc', str(main_doc), '-o', str(output_path)] + command.extend(['--bibliography', str(bib_file)]) + command.append('--citeproc') # Use the citation processor + + # Handle CSL style + style_key = task_key.split('_')[-1] # e.g., 'apa' from 'pdf_apa' + csl_path_or_url = None + + if csl_file: + logger.info(f"Using CSL file found in ZIP: {csl_file.name}") + csl_path_or_url = str(csl_file) + else: + # Look up CSL from config + try: + csl_path_or_url = app_config['academic_settings']['pandoc']['csl_files'][style_key] + logger.info(f"Using CSL style '{style_key}' from configuration.") + except KeyError: + logger.warning(f"No CSL style found for key '{style_key}'. Pandoc will use its default.") + + if csl_path_or_url: + command.extend(['--csl', csl_path_or_url]) + + command.extend(['--pdf-engine', 'xelatex']) + + update_job_status(db, job_id, "processing", progress=50) + logger.info(f"Executing Pandoc command for job {job_id}: {' '.join(command)}") + + # 4. Execute command directly to control working directory and error capture + try: + process = subprocess.run( + command, + capture_output=True, + text=True, + timeout=300, + check=True, # Raise CalledProcessError on non-zero exit + cwd=unzip_dir, # Run pandoc in the unzipped directory + preexec_fn=globals().get("_limit_resources_preexec", None) + ) + except subprocess.CalledProcessError as e: + # Capture the full, detailed error log from pandoc/latex + error_log = e.stderr or "No stderr output." + logger.error(f"Pandoc compilation failed. Full log:\n{error_log}") + # Raise a more informative exception for the user + raise Exception(f"Pandoc compilation failed. Please check your document for errors. Log: {error_log[:2000]}") from e + + # 5. Verify output + if not output_path.exists() or output_path.stat().st_size == 0: + raise Exception("Pandoc conversion failed: The tool produced an empty or missing output file.") + + mark_job_as_completed(db, job_id, output_filepath_str=str(output_path), preview="Successfully created academic PDF.") + logger.info(f"Academic Pandoc task for job {job_id} completed.") + + except Exception as e: + logger.exception(f"ERROR during academic Pandoc task for job {job_id}") + update_job_status(db, job_id, "failed", error=f"Pandoc task failed: {e}") + finally: + # 6. Cleanup + if unzip_dir.exists(): + shutil.rmtree(unzip_dir, ignore_errors=True) + try: + ensure_path_is_safe(input_path, [PATHS.UPLOADS_DIR, PATHS.CHUNK_TMP_DIR]) + input_path.unlink(missing_ok=True) + except Exception: + logger.exception("Failed to cleanup input ZIP file after Pandoc task.") + + if db: + db.close() + + # If this was a sub-job, trigger a progress update for its parent. + db_for_check = SessionLocal() + try: + job = get_job(db_for_check, job_id) + if job and job.parent_job_id: + _update_parent_zip_job_progress(job.parent_job_id) + finally: + db_for_check.close() + + send_webhook_notification(job_id, app_config, base_url) + + +@huey.task() +def _update_parent_zip_job_progress(parent_job_id: str): + """Checks and updates the progress of a parent 'unzip' job.""" + db = SessionLocal() + try: + parent_job = get_job(db, parent_job_id) + if not parent_job or parent_job.status not in ['processing', 'pending']: + return # Job is already finalized or doesn't exist + + child_jobs = db.query(Job).filter(Job.parent_job_id == parent_job.id).all() + total_children = len(child_jobs) + + if total_children == 0: + return # Should not happen if dispatched correctly, but safeguard. + + finished_children = 0 + for child in child_jobs: + if child.status in ['completed', 'failed', 'cancelled']: + finished_children += 1 + + progress = int((finished_children / total_children) * 100) if total_children > 0 else 100 + + if finished_children == total_children: + failed_count = sum(1 for child in child_jobs if child.status == 'failed') + preview = f"Batch processing complete. {total_children - failed_count}/{total_children} tasks succeeded." + if failed_count > 0: + preview += f" ({failed_count} failed)." + mark_job_as_completed(db, parent_job.id, preview=preview) + logger.info(f"Batch job {parent_job.id} marked as completed.") + else: + if parent_job.progress != progress: + update_job_status(db, parent_job.id, 'processing', progress=progress) + + except Exception as e: + logger.exception(f"Error in _update_parent_zip_job_progress for parent {parent_job_id}: {e}") + finally: + db.close() + + @huey.task() def unzip_and_dispatch_task(job_id: str, input_path_str: str, sub_task_type: str, sub_task_options: dict, user: dict, app_config: dict, base_url: str): db = SessionLocal() input_path = Path(input_path_str) unzip_dir = PATHS.UPLOADS_DIR / f"unzipped_{job_id}" + logger.info(f"Starting unzip and dispatch task for job {job_id} into {sub_task_type} jobs. ") + try: if not zipfile.is_zipfile(input_path): raise ValueError("Uploaded file is not a valid ZIP archive.") @@ -1886,61 +2097,23 @@ def unzip_and_dispatch_task(job_id: str, input_path_str: str, sub_task_type: str logger.exception("Failed to cleanup original ZIP file.") db.close() -@huey.periodic_task(crontab(minute='*/1')) # Runs every 1 minutes -def update_unzip_job_progress(): - """Periodically checks and updates the progress of parent 'unzip' jobs.""" - db = SessionLocal() - try: - # Find all 'unzip' jobs that are still marked as 'processing' - parent_jobs_to_check = db.query(Job).filter( - Job.task_type == 'unzip', - Job.status == 'processing' - ).all() - if not parent_jobs_to_check: - return # Nothing to do - - logger.info(f"Checking progress for {len(parent_jobs_to_check)} active batch jobs.") - - for parent_job in parent_jobs_to_check: - # Find all children of this parent job - child_jobs = db.query(Job).filter(Job.parent_job_id == parent_job.id).all() - total_children = len(child_jobs) - - if total_children == 0: - # This case shouldn't happen if unzip_and_dispatch_task works, but as a safeguard: - mark_job_as_completed(db, parent_job.id, preview="Batch job completed with no sub-tasks.") - continue - - finished_children = 0 - for child in child_jobs: - if child.status in ['completed', 'failed', 'cancelled']: - finished_children += 1 - - # Calculate and update progress - progress = int((finished_children / total_children) * 100) if total_children > 0 else 100 - - if finished_children == total_children: - # All children are done, mark the parent as completed - failed_count = sum(1 for child in child_jobs if child.status == 'failed') - preview = f"Batch processing complete. {total_children - failed_count}/{total_children} tasks succeeded." - if failed_count > 0: - preview += f" ({failed_count} failed)." - mark_job_as_completed(db, parent_job.id, preview=preview) - logger.info(f"Batch job {parent_job.id} marked as completed.") - else: - # Update the progress if it has changed - if parent_job.progress != progress: - update_job_status(db, parent_job.id, 'processing', progress=progress) - - except Exception as e: - logger.exception(f"Error in periodic task update_unzip_job_progress: {e}") - finally: - db.close() # -------------------------------------------------------------------------------- # --- 5. FASTAPI APPLICATION # -------------------------------------------------------------------------------- + +# --- SSE Broadcaster for real-time UI updates --- +import asyncio +import json + + + + + +# -------------------------------------------------------------------------------- +# --- 2. DATABASE & Schemas + async def download_kokoro_models_if_missing(): """Checks for Kokoro TTS model files and downloads them if they don't exist.""" files_to_download = { @@ -2001,6 +2174,13 @@ async def lifespan(app: FastAPI): load_app_config() + # Start the cache cleanup thread + global _cache_cleanup_thread + if _cache_cleanup_thread is None: + _cache_cleanup_thread = threading.Thread(target=_whisper_cache_cleanup_worker, daemon=True) + _cache_cleanup_thread.start() + logger.info("Whisper model cache cleanup thread started.") + # Download required models on startup if shutil.which("kokoro-tts"): await download_kokoro_models_if_missing() @@ -2049,6 +2229,14 @@ app.add_middleware( max_age=14 * 24 * 60 * 60 # 14 days ) +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], # Allows all origins + allow_credentials=True, + allow_methods=["*"], # Allows all methods + allow_headers=["*"], # Allows all headers +) + # Static / templates app.mount("/static", StaticFiles(directory=str(PATHS.BASE_DIR / "static")), name="static") @@ -2144,28 +2332,59 @@ async def upload_chunk( temp_dir.mkdir(exist_ok=True) chunk_path = temp_dir / f"{chunk_number}.chunk" - try: - with open(chunk_path, "wb") as buffer: - shutil.copyfileobj(chunk.file, buffer) - finally: - chunk.file.close() + def save_chunk_sync(): + try: + with open(chunk_path, "wb") as buffer: + shutil.copyfileobj(chunk.file, buffer) + finally: + chunk.file.close() + + await run_in_threadpool(save_chunk_sync) + return JSONResponse({"message": f"Chunk {chunk_number} for {safe_upload_id} uploaded."}) + async def _stitch_chunks(temp_dir: Path, final_path: Path, total_chunks: int): - """Stitches chunks together and cleans up.""" + """Stitches chunks together memory-efficiently and cleans up.""" ensure_path_is_safe(temp_dir, [PATHS.CHUNK_TMP_DIR]) ensure_path_is_safe(final_path, [PATHS.UPLOADS_DIR]) - with open(final_path, "wb") as final_file: - for i in range(total_chunks): - chunk_path = temp_dir / f"{i}.chunk" - if not chunk_path.exists(): - shutil.rmtree(temp_dir, ignore_errors=True) - raise HTTPException(status_code=400, detail=f"Upload failed: missing chunk {i}") - with open(chunk_path, "rb") as chunk_file: - final_file.write(chunk_file.read()) - shutil.rmtree(temp_dir, ignore_errors=True) -@app.post("/upload/finalize", status_code=status.HTTP_202_ACCEPTED) + # This is a blocking function that will be run in a threadpool + def do_stitch(): + with open(final_path, "wb") as final_file: + for i in range(total_chunks): + chunk_path = temp_dir / f"{i}.chunk" + if not chunk_path.exists(): + # Raise an exception that can be caught and handled + raise FileNotFoundError(f"Upload failed: missing chunk {i}") + with open(chunk_path, "rb") as chunk_file: + # Use copyfileobj for memory efficiency + shutil.copyfileobj(chunk_file, final_file) + + try: + await run_in_threadpool(do_stitch) + except FileNotFoundError as e: + # If a chunk was missing, clean up and re-raise as HTTPException + shutil.rmtree(temp_dir, ignore_errors=True) + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + # For any other error during stitching, clean up and re-raise + shutil.rmtree(temp_dir, ignore_errors=True) + raise e # Re-raise the original exception + else: + # If successful, clean up the temp directory + shutil.rmtree(temp_dir, ignore_errors=True) + +def _parse_tool_and_task_key(output_format: str, all_tool_keys: list) -> (str, str): + """Robustly parses an output_format string to find the matching tool and task key.""" + # Sort keys by length descending to match longest prefix first (e.g., 'ghostscript_image' before 'ghostscript') + for tool_key in sorted(all_tool_keys, key=len, reverse=True): + if output_format.startswith(tool_key + '_'): + task_key = output_format[len(tool_key) + 1:] + return tool_key, task_key + raise ValueError(f"Could not determine tool from output_format: {output_format}") + +@app.post("/upload/finalize", response_model=JobSchema, status_code=status.HTTP_202_ACCEPTED) async def finalize_upload(request: Request, payload: FinalizeUploadPayload, user: dict = Depends(require_user), db: Session = Depends(get_db)): safe_upload_id = secure_filename(payload.upload_id) temp_dir = ensure_path_is_safe(PATHS.CHUNK_TMP_DIR / safe_upload_id, [PATHS.CHUNK_TMP_DIR]) @@ -2183,7 +2402,22 @@ async def finalize_upload(request: Request, payload: FinalizeUploadPayload, user base_url = str(request.base_url) - if Path(safe_filename).suffix.lower() == '.zip': + # Check if the selected conversion is the new academic pandoc task + tool, task_key = None, None + if payload.task_type == 'conversion': + try: + all_tools = APP_CONFIG.get("conversion_tools", {}).keys() + tool, task_key = _parse_tool_and_task_key(payload.output_format, all_tools) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid or missing output_format for conversion.") + + if tool == 'pandoc_academic': + # This is a single job that processes a ZIP file as a project. + options = {"output_format": payload.output_format} + dispatch_single_file_job(payload.original_filename, str(final_path), "conversion", user, db, APP_CONFIG, base_url, job_id=job_id, options=options) + + elif Path(safe_filename).suffix.lower() == '.zip': + # This is the original batch processing logic for ZIP files. job_data = JobCreate( id=job_id, user_id=user['sub'], task_type="unzip", original_filename=payload.original_filename, input_filepath=str(final_path), @@ -2197,10 +2431,26 @@ async def finalize_upload(request: Request, payload: FinalizeUploadPayload, user } unzip_and_dispatch_task(job_id, str(final_path), payload.task_type, sub_task_options, user, APP_CONFIG, base_url) else: + # This is the logic for all other single-file uploads. options = {"model_size": payload.model_size, "model_name": payload.model_name, "output_format": payload.output_format} dispatch_single_file_job(payload.original_filename, str(final_path), payload.task_type, user, db, APP_CONFIG, base_url, job_id=job_id, options=options) - return {"job_id": job_id, "status": "pending"} + # --- FIX STARTS HERE --- + # Instead of returning a minimal object, fetch the newly created job + # from the database and return the full serialized object. This ensures + # the frontend has all the data it needs to correctly update the UI row. + db.flush() # Ensure the job is available to be queried + db_job = get_job(db, job_id) + if not db_job: + # This is an unlikely race condition but we handle it just in case. + # The SSE event will still create the row correctly. + raise HTTPException(status_code=500, detail="Job was created but could not be retrieved for an immediate response.") + + # Also, update the function signature to use the response_model + # from: @app.post("/upload/finalize", status_code=status.HTTP_202_ACCEPTED) + # to: @app.post("/upload/finalize", response_model=JobSchema, status_code=status.HTTP_202_ACCEPTED) + return db_job + # --- FIX ENDS HERE --- # --- LEGACY DIRECT-UPLOAD ROUTES (kept for compatibility) --- @@ -2545,25 +2795,20 @@ async def get_index(request: Request): @app.get("/settings") async def get_settings_page(request: Request): - """Displays the contents of the currently active configuration file.""" + """Displays the contents of the currently active configuration.""" user = get_current_user(request) admin_status = is_admin(request) - current_config, config_source = {}, "none" - try: - with open(PATHS.SETTINGS_FILE, 'r', encoding='utf8') as f: - current_config = yaml.safe_load(f) or {} + + # Use the globally loaded and merged APP_CONFIG for consistency + # This ensures all default keys are present before rendering. + current_config = APP_CONFIG + + # Determine the source file for display purposes + config_source = "none" + if PATHS.SETTINGS_FILE.exists(): config_source = str(PATHS.SETTINGS_FILE.name) - except FileNotFoundError: - try: - with open(PATHS.DEFAULT_SETTINGS_FILE, 'r', encoding='utf8') as f: - current_config = yaml.safe_load(f) or {} - config_source = str(PATHS.DEFAULT_SETTINGS_FILE.name) - except Exception as e: - logger.exception(f"CRITICAL: Could not load fallback config: {e}") - config_source = "error" - except Exception as e: - logger.exception(f"Could not load primary config: {e}") - config_source = "error" + elif PATHS.DEFAULT_SETTINGS_FILE.exists(): + config_source = str(PATHS.DEFAULT_SETTINGS_FILE.name) return templates.TemplateResponse( "settings.html", @@ -2671,6 +2916,26 @@ async def get_job_status(job_id: str, db: Session = Depends(get_db), user: dict raise HTTPException(status_code=404, detail="Job not found.") return job + + +class JobStatusRequest(BaseModel): + job_ids: TypingList[str] + +@app.post("/api/v1/jobs/status", response_model=TypingList[JobSchema]) +async def get_jobs_status(payload: JobStatusRequest, db: Session = Depends(get_db), user: dict = Depends(require_user)): + """ + Accepts a list of job IDs and returns their current status. + This is used by the frontend for polling active jobs. + """ + if not payload.job_ids: + return [] + + # Fetch all requested jobs from the database in a single query + jobs = db.query(Job).filter(Job.id.in_(payload.job_ids), Job.user_id == user['sub']).all() + return jobs + + + @app.get("/download/{filename}") async def download_file(filename: str, db: Session = Depends(get_db), user: dict = Depends(require_user)): safe_filename = secure_filename(filename) @@ -2730,15 +2995,15 @@ async def download_zip_batch(job_id: str, db: Session = Depends(get_db), user: d download_filename = f"{Path(job.original_filename).stem}{file_path.suffix}" zip_file.write(file_path, arcname=download_filename) files_added += 1 - + if files_added == 0: raise HTTPException(status_code=404, detail="No processed files found for the completed sub-jobs.") zip_buffer.seek(0) - + # Generate a filename for the download batch_filename = f"{Path(parent_job.original_filename).stem}_processed.zip" - + return StreamingResponse(zip_buffer, media_type="application/x-zip-compressed", headers={ 'Content-Disposition': f'attachment; filename="{batch_filename}"' }) @@ -2755,4 +3020,5 @@ async def health(): @app.get('/favicon.ico', include_in_schema=False) async def favicon(): - return FileResponse(str(PATHS.BASE_DIR / 'static' / 'favicon.png')) \ No newline at end of file + return FileResponse(str(PATHS.BASE_DIR / 'static' / 'favicon.png')) + diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 diff --git a/run.sh b/run.sh index cea70fa..73b3e69 100755 --- a/run.sh +++ b/run.sh @@ -9,11 +9,11 @@ SECRET_KEY= UPLOADS_DIR=./uploads PROCESSED_DIR=./processed # Start Gunicorn in the background -gunicorn -w 4 --threads 2 -k uvicorn.workers.UvicornWorker --forwarded-allow-ips='*' --error-logfile - --access-logfile - main:app -b 0.0.0.0:8000 & +gunicorn -w 4 --threads 2 -k uvicorn.workers.UvicornWorker --forwarded-allow-ips='*' --error-logfile - --access-logfile - main:app -b 0.0.0.0:8008 & echo "Started Gunicorn..." # Store the Gunicorn process ID GUNICORN_PID=$! echo "Gunicorn PID: $GUNICORN_PID" # Start the Huey consumer in the foreground exec huey_consumer.py main.huey -w 4 -echo "Started Huey consumer..." \ No newline at end of file +echo "Started Huey consumer..." diff --git a/settings.default.yml b/settings.default.yml old mode 100644 new mode 100755 index 5d8fc16..2168a2d --- a/settings.default.yml +++ b/settings.default.yml @@ -6,11 +6,11 @@ auth_settings: oidc_end_session_endpoint: https://accounts.example.com/oidc/session/end admin_users: - user@example.com -web_hook_settings: +webhook_settings: enabled: False - allow_chunked_api_uploads": False - allowed_callback_urls: - callback_bearer_token": + allow_chunked_api_uploads: False + allowed_callback_urls: [] + callback_bearer_token: tts_settings: piper: model_dir: "./models/tts" @@ -99,6 +99,13 @@ ocr_settings: clean: true optimize: 1 force_ocr: true +academic_settings: + pandoc: + csl_files: + apa: https://www.zotero.org/styles/apa + mla: https://www.zotero.org/styles/modern-language-association + chicago: https://www.zotero.org/styles/chicago-author-date + chicago-fullnote: https://www.zotero.org/styles/chicago-fullnote-bibliography transcription_settings: whisper: compute_type: int8 @@ -371,3 +378,12 @@ conversion_tools: jpg_q85: JPEG (High Quality) jpg_q75: JPEG (Web Quality) jpg_q60: JPEG (Aggressive Compression) + pandoc_academic: + name: Pandoc (Academic Document) + command_template: "pandoc {main_document} -o {output} --bibliography {bib_file} --citeproc --csl {csl_style}" + timeout: 300 + formats: + pdf_apa: "PDF with Bibliography (APA Style)" + pdf_mla: "PDF with Bibliography (MLA Style)" + pdf_chicago: "PDF with Bibliography (Chicago Style)" + pdf_chicago_fullnote: "PDF with Bibliography (Chicago Full Note)" diff --git a/static/css/settings.css b/static/css/settings.css old mode 100644 new mode 100755 index 3225230..d186582 --- a/static/css/settings.css +++ b/static/css/settings.css @@ -9,13 +9,8 @@ border-bottom: 1px solid var(--divider-color); } -.settings-header h1 {e - margin: 0 0 0.25rem 0; -} - -.settings-header p { +.settings-header h1 { margin: 0; - color: var(--muted-text); } .back-button { @@ -32,11 +27,18 @@ background-color: var(--primary-hover); } +/* Main layout grid for settings */ +.settings-main-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(400px, 1fr)); + gap: 2rem; +} + .settings-group { border: 1px solid var(--border-color); border-radius: 8px; padding: 1.5rem; - margin-bottom: 2rem; + margin-bottom: 2rem; /* Kept for spacing when grid stacks */ } .settings-group legend { @@ -70,36 +72,39 @@ .form-textarea { resize: vertical; min-height: 60px; - font-family: 'Courier New', Courier, monospace; + /* Use a more standard monospace font stack */ + font-family: Consolas, 'Courier New', Courier, monospace; } .field-description { - font-size: 0.85rem; + font-size: 0.9rem; color: var(--muted-text); - margin-top: -0.5rem; - margin-bottom: 1rem; + margin-top: 0.25rem; + margin-bottom: 0.75rem; + line-height: 1.4; } .field-description code { background-color: rgba(255,255,255,0.1); padding: 0.1rem 0.3rem; border-radius: 3px; - font-size: 0.8rem; + font-size: 0.85rem; } .checkbox-group { display: flex; align-items: center; gap: 0.75rem; - margin-bottom: 0.5rem; + margin-bottom: 0.75rem; } .checkbox-group input[type="checkbox"] { - width: 1rem; - height: 1rem; + width: 1.1rem; + height: 1.1rem; + accent-color: var(--primary-color); } .tool-grid { display: grid; - grid-template-columns: repeat(auto-fit, minmax(350px, 1fr)); + grid-template-columns: 1fr; /* Simplified to single column within a settings group */ gap: 1rem; } @@ -121,22 +126,23 @@ gap: 1rem; margin-top: 1.5rem; } + .button-primary { display: inline-block; background: var(--primary-color); - background-color: transparent; - border-color: var(--border-color); - border-width: 1px; - color: #ffffff; + color: var(--bg-color); + border: 1px solid var(--primary-color); padding: 0.65rem 1.5rem; font-size: 1rem; font-weight: 600; border-radius: 5px; cursor: pointer; - transition: background-color 0.15s ease; + transition: all 0.15s ease; } .button-primary:hover { background: var(--primary-hover); + color: var(--text-color); + border-color: var(--primary-hover); } .save-status { @@ -191,4 +197,16 @@ } .button-danger:hover { background-color: #ff8f8f; -} \ No newline at end of file +} + +/* Responsive adjustments */ +@media (max-width: 768px) { + .danger-action { + flex-direction: column; + align-items: flex-start; + } + .button-danger { + width: 100%; + text-align: center; + } +} diff --git a/static/css/style.css b/static/css/style.css old mode 100644 new mode 100755 index d2884c4..1aebb89 --- a/static/css/style.css +++ b/static/css/style.css @@ -18,7 +18,7 @@ --border-color: rgba(255, 255, 255, 0.1); --divider-color: rgba(255, 255, 255, 0.06); - --font-family: 'Inter', -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; + --font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; } /* Page */ diff --git a/static/css/style.old b/static/css/style.old old mode 100644 new mode 100755 diff --git a/static/favicon.ico b/static/favicon.ico old mode 100644 new mode 100755 diff --git a/static/favicon.png b/static/favicon.png old mode 100644 new mode 100755 diff --git a/static/js/script.js b/static/js/script.js old mode 100644 new mode 100755 index 06ac803..89262d7 --- a/static/js/script.js +++ b/static/js/script.js @@ -1,29 +1,14 @@ document.addEventListener('DOMContentLoaded', () => { // --- Constants --- const CHUNK_SIZE = 5 * 1024 * 1024; // 5 MB chunks - - // Allow server to provide API prefix (e.g. "/api/v1") via window.APP_CONFIG.api_base const API_BASE = (window.APP_CONFIG && window.APP_CONFIG.api_base) ? window.APP_CONFIG.api_base.replace(/\/$/, '') : ''; - function apiUrl(path) { - // path may start with or without a leading slash - if (!path) return API_BASE || '/'; - if (path.startsWith('/')) { - return `${API_BASE}${path}`; - } - return `${API_BASE}/${path}`; - } - - // --- User Locale and Timezone Detection --- + // --- User Locale --- const USER_LOCALE = navigator.language || 'en-US'; const USER_TIMEZONE = Intl.DateTimeFormat().resolvedOptions().timeZone; const DATETIME_FORMAT_OPTIONS = { - year: 'numeric', - month: 'short', - day: 'numeric', - hour: 'numeric', - minute: '2-digit', - timeZone: USER_TIMEZONE, + year: 'numeric', month: 'short', day: 'numeric', + hour: 'numeric', minute: '2-digit', timeZone: USER_TIMEZONE, }; // --- Element Selectors --- @@ -65,32 +50,30 @@ document.addEventListener('DOMContentLoaded', () => { const dialogOutputFormatSelect = document.getElementById('dialog-output-format-select'); const dialogTtsModelSelect = document.getElementById('dialog-tts-model-select'); - // --- State Variables --- let conversionChoices = null; let transcriptionChoices = null; let ttsChoices = null; let dialogConversionChoices = null; let dialogTtsChoices = null; - let ttsModelsCache = []; // Cache for formatted TTS models list - const activePolls = new Map(); + let ttsModelsCache = []; let stagedFiles = null; + let jobPollerInterval = null; // Polling timer + const POLLING_INTERVAL_MS = 1000; // Check for updates every 3 seconds + // --- Core Functions --- + + function apiUrl(path) { + if (!path) return API_BASE || '/'; + return path.startsWith('/') ? `${API_BASE}${path}` : `${API_BASE}/${path}`; + } - // --- Authentication-aware Fetch Wrapper --- async function authFetch(url, options = {}) { - // Normalize URL through apiUrl() if a bare endpoint is provided if (typeof url === 'string' && url.startsWith('/')) { url = apiUrl(url); } - - // Add default options: include credentials and accept JSON by default - options = Object.assign({}, options); - if (!Object.prototype.hasOwnProperty.call(options, 'credentials')) { - options.credentials = 'include'; - } - options.headers = options.headers || {}; - if (!options.headers.Accept) options.headers.Accept = 'application/json'; + options = { credentials: 'include', ...options }; + options.headers = { Accept: 'application/json', ...options.headers }; const response = await fetch(url, options); if (response.status === 401) { @@ -101,8 +84,6 @@ document.addEventListener('DOMContentLoaded', () => { return response; } - - // --- Helper Functions --- function formatBytes(bytes, decimals = 1) { if (!+bytes) return '0 Bytes'; const k = 1024; @@ -112,22 +93,165 @@ document.addEventListener('DOMContentLoaded', () => { return `${parseFloat((bytes / Math.pow(k, i)).toFixed(dm))} ${sizes[i]}`; } - // --- Chunked Uploading Logic --- + async function pollForJobUpdates() { + try { + const allJobs = await authFetch('/jobs').then(res => res.json()); + + const topLevelJobs = []; + const childJobs = []; + + allJobs.forEach(job => { + if (job.parent_job_id) { + childJobs.push(job); + } else { + topLevelJobs.push(job); + } + }); + + // Render top-level jobs first, then child jobs. + // The renderJobRow function handles both creating and updating rows. + topLevelJobs.forEach(job => renderJobRow(job)); + childJobs.forEach(job => renderJobRow(job)); + + // Stop polling if there are no more active jobs. + const hasActiveJobs = allJobs.some(job => ['pending', 'processing', 'uploading'].includes(job.status)); + if (!hasActiveJobs) { + stopJobPolling(); + } + } catch (error) { + console.error("Job polling failed:", error); + // Don't stop polling on error, just log it and retry next interval. + } + } + + function startJobPolling() { + if (jobPollerInterval) return; // Poller is already running + // Run once after a short delay, then start the regular interval + setTimeout(pollForJobUpdates, 1000); + jobPollerInterval = setInterval(pollForJobUpdates, POLLING_INTERVAL_MS); + } + + function stopJobPolling() { + if (jobPollerInterval) { + clearInterval(jobPollerInterval); + jobPollerInterval = null; + } + } + + function renderJobRow(job) { + const permanentDomId = `job-${job.id}`; + let row = document.getElementById(permanentDomId); + + // --- Generate Content --- + let taskTypeLabel = job.task_type; + if (job.task_type === 'conversion' && job.processed_filepath) { + const extension = job.processed_filepath.split('.').pop(); + taskTypeLabel = `Convert to ${extension.toUpperCase()}`; + } else if (job.task_type === 'academic_pandoc') { + taskTypeLabel = 'Academic PDF'; + } else if (job.task_type === 'tts') { + taskTypeLabel = 'Synthesize Speech'; + } else if (job.task_type === 'unzip') { + taskTypeLabel = 'Unpack ZIP'; + } else if (job.task_type) { + taskTypeLabel = job.task_type.charAt(0).toUpperCase() + job.task_type.slice(1); + } + const formattedDate = new Date(job.created_at).toLocaleString(USER_LOCALE, DATETIME_FORMAT_OPTIONS); + let statusHtml = `${job.status}`; + if ((job.status === 'processing' || job.status === 'pending') && job.task_type === 'unzip') { + statusHtml += `
`; + } else if (job.status === 'processing') { + const progressClass = (job.progress > 0) ? '' : 'indeterminate'; + const progressWidth = (job.progress > 0) ? job.progress : 100; + statusHtml += `
`; + } + let actionHtml = '-'; + if (['pending', 'processing', 'uploading'].includes(job.status)) { + actionHtml = ``; + } else if (job.status === 'completed') { + if (job.task_type === 'unzip') { + actionHtml = `Download Batch`; + } else if (job.processed_filepath) { + const downloadFilename = job.processed_filepath.split(/[\\/]/).pop(); + actionHtml = `Download`; + } + } else if (job.status === 'failed') { + const errorTitle = job.error_message ? ` title="${job.error_message.replace(/"/g, '"')}"` : ''; + actionHtml = `Failed`; + } else if (job.status === 'cancelled') { + actionHtml = `Cancelled`; + } + let fileSizeHtml = job.input_filesize ? formatBytes(job.input_filesize) : '-'; + if (job.status === 'completed' && job.output_filesize) { + fileSizeHtml += ` → ${formatBytes(job.output_filesize)}`; + } + let checkboxHtml = ''; + if (job.status === 'completed' && job.processed_filepath && job.task_type !== 'unzip') { + checkboxHtml = ``; + } + + // --- Create or Update logic --- + if (row) { + // UPDATE an existing row + row.querySelector('td[data-label="Select"] .cell-value').innerHTML = checkboxHtml; + row.querySelector('td[data-label="File Size"] .cell-value').innerHTML = fileSizeHtml; + row.querySelector('td[data-label="Task"] .cell-value').innerHTML = taskTypeLabel; + row.querySelector('td[data-label="Status"] .cell-value').innerHTML = statusHtml; + row.querySelector('td[data-label="Action"] .cell-value').innerHTML = actionHtml; + } else { + // CREATE a new row + row = document.createElement('tr'); + row.id = permanentDomId; + const escapedFilename = job.original_filename ? job.original_filename.replace(//g, ">") : "No filename"; + const rowClasses = []; + if (job.parent_job_id) rowClasses.push('sub-job'); + if (job.task_type === 'unzip') rowClasses.push('parent-job'); + row.className = rowClasses.join(' '); + if (job.parent_job_id) row.dataset.parentId = job.parent_job_id; + const expanderHtml = job.task_type === 'unzip' ? '' : ''; + + row.innerHTML = ` + ${checkboxHtml} + ${expanderHtml}${escapedFilename} + ${fileSizeHtml} + ${taskTypeLabel} + ${formattedDate} + ${statusHtml} + ${actionHtml} + `; + const parentRow = job.parent_job_id ? document.getElementById(`job-${job.parent_job_id}`) : null; + if (parentRow) { + parentRow.parentNode.insertBefore(row, parentRow.nextSibling); + } else { + jobListBody.prepend(row); + } + } + } + async function uploadFileInChunks(file, taskType, options = {}) { const uploadId = 'upload-' + Date.now() + '-' + Math.random().toString(36).substr(2, 9); const totalChunks = Math.ceil(file.size / CHUNK_SIZE); - const preliminaryJob = { - id: uploadId, - status: 'uploading', - progress: 0, - original_filename: file.name, - input_filesize: file.size, - task_type: taskType, - created_at: new Date().toISOString() - }; - renderJobRow(preliminaryJob); + // Manually create and insert the temporary "uploading" row. + const tempRow = document.createElement('tr'); + tempRow.id = uploadId; + const escapedFilename = file.name.replace(//g, ">"); + const taskLabel = taskType.charAt(0).toUpperCase() + taskType.slice(1); + tempRow.innerHTML = ` + - + ${escapedFilename} + ${formatBytes(file.size)} + ${taskLabel} + ${new Date().toLocaleString(USER_LOCALE, DATETIME_FORMAT_OPTIONS)} + + uploading +
+ + - + `; + jobListBody.prepend(tempRow); + // Upload chunks and update the progress bar directly. for (let chunkNumber = 0; chunkNumber < totalChunks; chunkNumber++) { const start = chunkNumber * CHUNK_SIZE; const end = Math.min(start + CHUNK_SIZE, file.size); @@ -138,146 +262,69 @@ document.addEventListener('DOMContentLoaded', () => { formData.append('chunk_number', chunkNumber); try { - const response = await authFetch('/upload/chunk', { - method: 'POST', - body: formData, - }); - if (!response.ok) { - throw new Error(`Chunk upload failed with status: ${response.status}`); - } + const response = await authFetch('/upload/chunk', { method: 'POST', body: formData }); + if (!response.ok) throw new Error(`Chunk upload failed: ${response.statusText}`); const progress = Math.round(((chunkNumber + 1) / totalChunks) * 100); - updateUploadProgress(uploadId, progress); + tempRow.querySelector('.progress-bar').style.width = `${progress}%`; } catch (error) { - console.error(`Error uploading chunk ${chunkNumber} for ${file.name}:`, error); - if (error.message !== 'Session expired') { - updateJobToFailedState(uploadId, `Upload failed: ${error.message}`); - } - return; + console.error(`Error uploading chunk ${chunkNumber}:`, error); + tempRow.querySelector('.status-cell-value').innerHTML = `Upload Failed`; + return; // Stop the upload process } } + // Finalize the upload. try { - const finalizePayload = { - upload_id: uploadId, - original_filename: file.name, - total_chunks: totalChunks, - task_type: taskType, - ...options - }; + const finalizePayload = { upload_id: uploadId, original_filename: file.name, total_chunks: totalChunks, task_type: taskType, ...options }; const finalizeResponse = await authFetch('/upload/finalize', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(finalizePayload), + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(finalizePayload), }); if (!finalizeResponse.ok) { - let errorData = {}; - try { errorData = await finalizeResponse.json(); } catch (e) {} + const errorData = await finalizeResponse.json().catch(() => ({})); throw new Error(errorData.detail || 'Finalization failed'); } const result = await finalizeResponse.json(); - const uploadRow = document.getElementById(uploadId); - if (uploadRow) { - uploadRow.id = `job-${result.job_id}`; - const statusCell = uploadRow.querySelector('td[data-label="Status"] .cell-value'); - if (statusCell) { - statusCell.innerHTML = `Pending`; - } - } - startPolling(result.job_id); + + tempRow.remove(); + renderJobRow(result); + startJobPolling(); + } catch (error) { - console.error(`Error finalizing upload for ${file.name}:`, error); - if (error.message !== 'Session expired') { - updateJobToFailedState(uploadId, `Finalization failed: ${error.message}`); - } + console.error(`Error finalizing upload:`, error); + tempRow.querySelector('.status-cell-value').innerHTML = `Finalization Failed`; } } - function updateUploadProgress(uploadId, progress) { - const row = document.getElementById(uploadId); - if (row) { - const progressBar = row.querySelector('.progress-bar'); - if (progressBar) { - progressBar.style.width = `${progress}%`; - } - } - } - - function updateJobToFailedState(jobId, errorMessage) { - const row = document.getElementById(jobId); - if (row) { - const statusCell = row.querySelector('td[data-label="Status"] .cell-value'); - const actionCell = row.querySelector('td[data-label="Action"] .cell-value'); - if (statusCell) statusCell.innerHTML = `Failed`; - if (actionCell) { - const errorTitle = errorMessage ? ` title="${errorMessage.replace(/"/g, '"')}"` : ''; - actionCell.innerHTML = `Failed`; - } - } - } - - // --- Centralized Task Request Handler --- async function handleTaskRequest(taskType) { - if (mainFileInput.files.length === 0) { - alert('Please choose one or more files first.'); - return; - } - + if (mainFileInput.files.length === 0) return alert('Please choose one or more files first.'); const files = Array.from(mainFileInput.files); const options = {}; if (taskType === 'conversion') { const selectedFormat = conversionChoices.getValue(true); - if (!selectedFormat) { - alert('Please select a format to convert to.'); - return; - } + if (!selectedFormat) return alert('Please select a format to convert to.'); options.output_format = selectedFormat; } else if (taskType === 'transcription') { - const selectedModel = transcriptionChoices.getValue(true); - options.model_size = selectedModel; + options.model_size = transcriptionChoices.getValue(true); } else if (taskType === 'tts') { const selectedModel = ttsChoices.getValue(true); - if (!selectedModel) { - alert('Please select a voice model.'); - return; - } + if (!selectedModel) return alert('Please select a voice model.'); options.model_name = selectedModel; } - - // Disable buttons during upload process - startConversionBtn.disabled = true; - startOcrBtn.disabled = true; - startTranscriptionBtn.disabled = true; - startTtsBtn.disabled = true; - - const uploadPromises = files.map(file => uploadFileInChunks(file, taskType, options)); - await Promise.allSettled(uploadPromises); - - // Reset file input and re-enable buttons - mainFileInput.value = ''; // Resets the file list + [startConversionBtn, startOcrBtn, startTranscriptionBtn, startTtsBtn].forEach(btn => btn.disabled = true); + await Promise.allSettled(files.map(file => uploadFileInChunks(file, taskType, options))); + mainFileInput.value = ''; updateFileName(mainFileInput, mainFileName); - startConversionBtn.disabled = false; - startOcrBtn.disabled = false; - startTranscriptionBtn.disabled = false; - startTtsBtn.disabled = false; + [startConversionBtn, startOcrBtn, startTranscriptionBtn, startTtsBtn].forEach(btn => btn.disabled = false); } - function setupDragAndDropListeners() { let dragCounter = 0; - window.addEventListener('dragenter', (e) => { - e.preventDefault(); - dragCounter++; - document.body.classList.add('dragging'); - }); - window.addEventListener('dragleave', (e) => { - e.preventDefault(); - dragCounter--; - if (dragCounter === 0) document.body.classList.remove('dragging'); - }); - window.addEventListener('dragover', (e) => e.preventDefault()); - window.addEventListener('drop', (e) => { + window.addEventListener('dragenter', e => { e.preventDefault(); dragCounter++; document.body.classList.add('dragging'); }); + window.addEventListener('dragleave', e => { e.preventDefault(); dragCounter--; if (dragCounter === 0) document.body.classList.remove('dragging'); }); + window.addEventListener('dragover', e => e.preventDefault()); + window.addEventListener('drop', e => { e.preventDefault(); dragCounter = 0; document.body.classList.remove('dragging'); @@ -292,22 +339,12 @@ document.addEventListener('DOMContentLoaded', () => { function showActionDialog() { dialogFileCount.textContent = stagedFiles.length; - - // Setup Conversion Dropdown dialogOutputFormatSelect.innerHTML = mainOutputFormatSelect.innerHTML; if (dialogConversionChoices) dialogConversionChoices.destroy(); - dialogConversionChoices = new Choices(dialogOutputFormatSelect, { - searchEnabled: true, itemSelectText: 'Select', shouldSort: false, placeholder: true, placeholderValue: 'Select a format...', - }); - - // Setup TTS Dropdown + dialogConversionChoices = new Choices(dialogOutputFormatSelect, { searchEnabled: true, itemSelectText: 'Select', shouldSort: false, placeholder: true, placeholderValue: 'Select a format...' }); if (dialogTtsChoices) dialogTtsChoices.destroy(); - dialogTtsChoices = new Choices(dialogTtsModelSelect, { - searchEnabled: true, itemSelectText: 'Select', shouldSort: false, placeholder: true, placeholderValue: 'Select a voice...', - }); + dialogTtsChoices = new Choices(dialogTtsModelSelect, { searchEnabled: true, itemSelectText: 'Select', shouldSort: false, placeholder: true, placeholderValue: 'Select a voice...' }); dialogTtsChoices.setChoices(ttsModelsCache, 'value', 'label', true); - - dialogInitialView.style.display = 'grid'; dialogConvertView.style.display = 'none'; dialogTtsView.style.display = 'none'; @@ -317,161 +354,87 @@ document.addEventListener('DOMContentLoaded', () => { function closeActionDialog() { actionDialog.classList.remove('visible'); stagedFiles = null; - if (dialogConversionChoices) { - dialogConversionChoices.destroy(); - dialogConversionChoices = null; - } - if (dialogTtsChoices) { - dialogTtsChoices.destroy(); - dialogTtsChoices = null; - } + if (dialogConversionChoices) { dialogConversionChoices.destroy(); dialogConversionChoices = null; } + if (dialogTtsChoices) { dialogTtsChoices.destroy(); dialogTtsChoices = null; } } - - // --- Dialog Button Listeners --- - dialogConvertBtn.addEventListener('click', () => { - dialogInitialView.style.display = 'none'; - dialogConvertView.style.display = 'block'; - }); - dialogTtsBtn.addEventListener('click', () => { - dialogInitialView.style.display = 'none'; - dialogTtsView.style.display = 'block'; - }); - dialogBackBtn.addEventListener('click', () => { - dialogInitialView.style.display = 'grid'; - dialogConvertView.style.display = 'none'; - }); - dialogBackTtsBtn.addEventListener('click', () => { - dialogInitialView.style.display = 'grid'; - dialogTtsView.style.display = 'none'; - }); - dialogStartConversionBtn.addEventListener('click', () => handleDialogAction('conversion')); - dialogStartTtsBtn.addEventListener('click', () => handleDialogAction('tts')); - dialogOcrBtn.addEventListener('click', () => handleDialogAction('ocr')); - dialogTranscribeBtn.addEventListener('click', () => handleDialogAction('transcription')); - dialogCancelBtn.addEventListener('click', closeActionDialog); function handleDialogAction(action) { if (!stagedFiles) return; let options = {}; if (action === 'conversion') { const selectedFormat = dialogConversionChoices.getValue(true); - if (!selectedFormat) { - alert('Please select a format to convert to.'); - return; - } + if (!selectedFormat) return alert('Please select a format to convert to.'); options.output_format = selectedFormat; } else if (action === 'transcription') { options.model_size = mainModelSizeSelect.value; } else if (action === 'tts') { const selectedModel = dialogTtsChoices.getValue(true); - if (!selectedModel) { - alert('Please select a voice model.'); - return; - } + if (!selectedModel) return alert('Please select a voice model.'); options.model_name = selectedModel; } Array.from(stagedFiles).forEach(file => uploadFileInChunks(file, action, options)); closeActionDialog(); } - // ----------------------- - // TTS models loader (robust) - // ----------------------- async function loadTtsModels() { try { - const response = await authFetch('/api/v1/tts-voices'); - if (!response.ok) throw new Error('Failed to fetch TTS voices.'); - const voicesData = await response.json(); - - // voicesData might be an object map { id: meta } or an array [{ id, name, language, ... }] + const voicesData = await authFetch('/api/v1/tts-voices').then(res => res.json()); const voicesArray = []; if (Array.isArray(voicesData)) { - for (const v of voicesData) { - // Accept either { id, name, language } or { voice_id, title, locale } - const id = v.id || v.voice_id || v.voice || v.name || null; - const name = v.name || v.title || v.display_name || id || 'Unknown'; - const lang = (v.language && (v.language.name_native || v.language.name)) || v.locale || (id ? id.split(/[_-]/)[0] : 'Unknown'); - if (id) voicesArray.push({ id, name, lang }); - } + voicesData.forEach(v => { + const id = v.id || v.voice_id || v.name; + if (id) voicesArray.push({ id, name: v.name || id, lang: (v.language && v.language.name) || v.locale || id.split(/[_-]/)[0] }); + }); } else if (voicesData && typeof voicesData === 'object') { - for (const key in voicesData) { - if (!Object.prototype.hasOwnProperty.call(voicesData, key)) continue; + Object.keys(voicesData).forEach(key => { const v = voicesData[key]; const id = v.id || key; - const name = v.name || v.title || v.display_name || id; - const lang = (v.language && (v.language.name_native || v.language.name)) || v.locale || (id ? id.split(/[_-]/)[0] : 'Unknown'); - voicesArray.push({ id, name, lang }); - } - } else { - throw new Error('Unexpected voices payload'); - } - - // Group by language - const groups = {}; - for (const v of voicesArray) { - const langLabel = v.lang || 'Unknown'; - if (!groups[langLabel]) { - groups[langLabel] = { label: langLabel, id: langLabel, disabled: false, choices: [] }; - } - groups[langLabel].choices.push({ - value: v.id, - label: `${v.name}` + voicesArray.push({ id, name: v.name || id, lang: (v.language && v.language.name) || v.locale || id.split(/[_-]/)[0] }); }); } - ttsModelsCache = Object.values(groups).sort((a,b) => a.label.localeCompare(b.label)); - // If ttsChoices exists, update it; otherwise the initializer will set choices - if (ttsChoices) { - ttsChoices.setChoices(ttsModelsCache, 'value', 'label', true); - } + const groups = voicesArray.reduce((acc, v) => { + const langLabel = v.lang || 'Unknown'; + if (!acc[langLabel]) acc[langLabel] = { label: langLabel, choices: [] }; + acc[langLabel].choices.push({ value: v.id, label: v.name }); + return acc; + }, {}); + ttsModelsCache = Object.values(groups).sort((a, b) => a.label.localeCompare(b.label)); + if (ttsChoices) ttsChoices.setChoices(ttsModelsCache, 'value', 'label', true); } catch (error) { console.error("Couldn't load TTS voices:", error); - if (error.message !== 'Session expired') { - if (ttsChoices) { - ttsChoices.setChoices([{ value: '', label: 'Error loading voices', disabled: true }], 'value', 'label'); - } - } + if (ttsChoices && error.message !== 'Session expired') ttsChoices.setChoices([{ value: '', label: 'Error loading voices', disabled: true }], 'value', 'label'); } } - - function initializeSelectors() { - if (conversionChoices) conversionChoices.destroy(); - conversionChoices = new Choices(mainOutputFormatSelect, { - searchEnabled: true, itemSelectText: 'Select', shouldSort: false, placeholder: true, placeholderValue: 'Select a format...', - }); - const tools = window.APP_CONFIG.conversionTools || {}; - const choicesArray = []; - for (const toolKey in tools) { - const tool = tools[toolKey]; - const group = { label: tool.name, id: toolKey, disabled: false, choices: [] }; - for (const formatKey in tool.formats) { - group.choices.push({ value: `${toolKey}_${formatKey}`, label: `${tool.name} - ${formatKey.toUpperCase()} (${tool.formats[formatKey]})` }); - } - choicesArray.push(group); - } - conversionChoices.setChoices(choicesArray, 'value', 'label', true); - if (transcriptionChoices) transcriptionChoices.destroy(); - transcriptionChoices = new Choices(mainModelSizeSelect, { - searchEnabled: false, shouldSort: false, itemSelectText: '', - }); +function initializeSelectors() { + if (conversionChoices) conversionChoices.destroy(); + conversionChoices = new Choices(mainOutputFormatSelect, { searchEnabled: true, itemSelectText: 'Select', shouldSort: false, placeholder: true, placeholderValue: 'Select a format...' }); + const tools = window.APP_CONFIG.conversionTools || {}; + const choicesArray = Object.keys(tools).map(toolKey => { + const tool = tools[toolKey]; + return { + label: tool.name, + choices: Object.keys(tool.formats).map(formatKey => ({ + value: `${toolKey}_${formatKey}`, + // --- THIS IS THE MODIFIED LINE --- + label: `${tool.name} - ${tool.formats[formatKey]}` + })) + }; + }); + conversionChoices.setChoices(choicesArray, 'value', 'label', true); - if (ttsChoices) ttsChoices.destroy(); - ttsChoices = new Choices(mainTtsModelSelect, { - searchEnabled: true, itemSelectText: 'Select', shouldSort: false, placeholder: true, placeholderValue: 'Select voice...', - }); - loadTtsModels(); - } + if (transcriptionChoices) transcriptionChoices.destroy(); + transcriptionChoices = new Choices(mainModelSizeSelect, { searchEnabled: false, shouldSort: false, itemSelectText: '' }); + + if (ttsChoices) ttsChoices.destroy(); + ttsChoices = new Choices(mainTtsModelSelect, { searchEnabled: true, itemSelectText: 'Select', shouldSort: false, placeholder: true, placeholderValue: 'Select voice...' }); + loadTtsModels(); +} function updateFileName(input, nameDisplay) { const numFiles = input.files.length; - let displayText = numFiles === 1 ? input.files[0].name : `${numFiles} files selected`; - let displayTitle = numFiles > 1 ? Array.from(input.files).map(f => f.name).join(', ') : displayText; - if (numFiles === 0) { - displayText = 'No file chosen'; - displayTitle = 'No file chosen'; - } - nameDisplay.textContent = displayText; - nameDisplay.title = displayTitle; + nameDisplay.textContent = numFiles === 1 ? input.files[0].name : (numFiles > 1 ? `${numFiles} files selected` : 'No files chosen'); + nameDisplay.title = numFiles > 1 ? Array.from(input.files).map(f => f.name).join(', ') : nameDisplay.textContent; } async function handleCancelJob(jobId) { @@ -479,20 +442,12 @@ document.addEventListener('DOMContentLoaded', () => { try { const response = await authFetch(`/job/${jobId}/cancel`, { method: 'POST' }); if (!response.ok) { - let errorData = {}; - try { errorData = await response.json(); } catch (e) {} + const errorData = await response.json().catch(() => ({})); throw new Error(errorData.detail || 'Failed to cancel job.'); } - stopPolling(jobId); - const row = document.getElementById(`job-${jobId}`); - if (row) { - const statusCell = row.querySelector('td[data-label="Status"] .cell-value'); - const actionCell = row.querySelector('td[data-label="Action"] .cell-value'); - if (statusCell) statusCell.innerHTML = `Cancelled`; - if (actionCell) actionCell.innerHTML = `-`; - } + // Trigger a poll soon to see the "cancelled" status updated in the UI. + setTimeout(pollForJobUpdates, 500); } catch (error) { - console.error('Error cancelling job:', error); if (error.message !== 'Session expired') alert(`Error: ${error.message}`); } } @@ -500,34 +455,27 @@ document.addEventListener('DOMContentLoaded', () => { function handleSelectionChange() { const selectedCheckboxes = jobListBody.querySelectorAll('.job-checkbox:checked'); downloadSelectedBtn.disabled = selectedCheckboxes.length === 0; - - const allCheckboxes = jobListBody.querySelectorAll('.job-checkbox'); - selectAllJobsCheckbox.checked = allCheckboxes.length > 0 && selectedCheckboxes.length === allCheckboxes.length; + selectAllJobsCheckbox.checked = jobListBody.querySelectorAll('.job-checkbox').length > 0 && selectedCheckboxes.length === jobListBody.querySelectorAll('.job-checkbox').length; } async function handleBatchDownload() { const selectedIds = Array.from(jobListBody.querySelectorAll('.job-checkbox:checked')).map(cb => cb.value); if (selectedIds.length === 0) return; - downloadSelectedBtn.disabled = true; downloadSelectedBtn.textContent = 'Zipping...'; - try { const response = await authFetch('/download/batch', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ job_ids: selectedIds }) + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ job_ids: selectedIds }) }); if (!response.ok) throw new Error('Batch download failed.'); - const blob = await response.blob(); const url = window.URL.createObjectURL(blob); const a = document.createElement('a'); - a.style.display = 'none'; a.href = url; a.download = `file-wizard-batch-${Date.now()}.zip`; document.body.appendChild(a); a.click(); + document.body.removeChild(a); window.URL.revokeObjectURL(url); } catch (error) { console.error("Batch download error:", error); @@ -542,242 +490,68 @@ document.addEventListener('DOMContentLoaded', () => { try { const response = await authFetch('/jobs'); if (!response.ok) throw new Error('Failed to fetch jobs.'); - let jobs = await response.json(); - - // Sort jobs so parents come before children - jobs.sort((a, b) => { - if (a.id === b.parent_job_id) return -1; - if (b.id === a.parent_job_id) return 1; - return new Date(b.created_at) - new Date(a.created_at); - }); - + const jobs = await response.json(); jobListBody.innerHTML = ''; - for (const job of jobs.reverse()) { - renderJobRow(job); - if (['pending', 'processing'].includes(job.status)) startPolling(job.id); - } + jobs.sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); // Sort descending + jobs.reverse().forEach(renderJobRow); handleSelectionChange(); + startJobPolling(); } catch (error) { console.error("Couldn't load job history:", error); - if (error.message !== 'Session expired') { - jobListBody.innerHTML = 'Could not load job history.'; - } - } - } - - // --- Polling and UI Rendering --- - async function fetchAndRenderSubJobs(parentJobId) { - try { - const response = await authFetch('/jobs'); - if (!response.ok) return; - const allJobs = await response.json(); - const childJobs = allJobs.filter(j => j.parent_job_id === parentJobId); - - for (const childJob of childJobs) { - const childRowId = `job-${childJob.id}`; - if (!document.getElementById(childRowId)) { - renderJobRow(childJob); - if (['pending', 'processing'].includes(childJob.status)) { - startPolling(childJob.id); - } - } - } - } catch (error) { - console.error(`Failed to fetch sub-jobs for parent ${parentJobId}:`, error); + if (error.message !== 'Session expired') jobListBody.innerHTML = 'Could not load job history.'; } } - function startPolling(jobId) { - if (activePolls.has(jobId)) return; - - const pollLogic = async () => { - try { - const response = await authFetch(`/job/${jobId}`); - if (!response.ok) { - if (response.status === 404) stopPolling(jobId); - return; - } - const job = await response.json(); - renderJobRow(job); - - if (job.task_type === 'unzip' && job.status === 'processing') { - await fetchAndRenderSubJobs(job.id); - } - - if (['completed', 'failed', 'cancelled'].includes(job.status)) { - stopPolling(jobId); - } - } catch (error) { - console.error(`Error polling for job ${jobId}:`, error); - stopPolling(jobId); - } - }; - - const intervalId = setInterval(pollLogic, 3000); - activePolls.set(jobId, intervalId); - pollLogic(); // Run once immediately - } - - - function stopPolling(jobId) { - if (activePolls.has(jobId)) { - clearInterval(activePolls.get(jobId)); - activePolls.delete(jobId); - } - } - - function renderJobRow(job) { - const rowId = job.id && String(job.id).startsWith('upload-') ? job.id : `job-${job.id}`; - let row = document.getElementById(rowId); - if (!row) { - row = document.createElement('tr'); - row.id = rowId; - const parentRow = job.parent_job_id ? document.getElementById(`job-${job.parent_job_id}`) : null; - if (parentRow) { - parentRow.parentNode.insertBefore(row, parentRow.nextSibling); - } else { - jobListBody.prepend(row); - } - } - - let taskTypeLabel = job.task_type; - if (job.task_type === 'conversion' && job.processed_filepath) { - const extension = job.processed_filepath.split('.').pop(); - taskTypeLabel = `Convert to ${extension.toUpperCase()}`; - } else if (job.task_type === 'tts') { - taskTypeLabel = 'Synthesize Speech'; - } else if (job.task_type === 'unzip') { - taskTypeLabel = 'Unpack ZIP'; - } else if (job.task_type) { - taskTypeLabel = job.task_type.charAt(0).toUpperCase() + job.task_type.slice(1); - } - - const submittedDate = job.created_at ? new Date(job.created_at) : new Date(); - const formattedDate = submittedDate.toLocaleString(USER_LOCALE, DATETIME_FORMAT_OPTIONS); - - let statusHtml = `${job.status}`; - if (job.status === 'uploading' || (job.status === 'processing' && job.task_type === 'unzip')) { - statusHtml += `
`; - } else if (job.status === 'processing') { - const progressClass = (job.progress > 0) ? '' : 'indeterminate'; - const progressWidth = (job.progress > 0) ? job.progress : 100; - statusHtml += `
`; - } - - let actionHtml = `-`; - if (['pending', 'processing', 'uploading'].includes(job.status)) { - actionHtml = ``; - } else if (job.status === 'completed') { - if (job.task_type === 'unzip') { - actionHtml = `Download Batch`; - } else if (job.processed_filepath) { - const downloadFilename = job.processed_filepath.split(/[\\/]/).pop(); - actionHtml = `Download`; - } - } else if (job.status === 'failed') { - const errorTitle = job.error_message ? ` title="${job.error_message.replace(/"/g, '"')}"` : ''; - actionHtml = `Failed`; - } - - let fileSizeHtml = job.input_filesize ? formatBytes(job.input_filesize) : '-'; - if (job.status === 'completed' && job.output_filesize) { - fileSizeHtml += ` → ${formatBytes(job.output_filesize)}`; - } - - const escapedFilename = job.original_filename ? job.original_filename.replace(//g, ">") : "No filename"; - - let checkboxHtml = ''; - if (job.status === 'completed' && job.processed_filepath && job.task_type !== 'unzip') { - checkboxHtml = ``; - } - - const rowClasses = []; - if(job.parent_job_id) rowClasses.push('sub-job'); - if(job.task_type === 'unzip') rowClasses.push('parent-job'); - row.className = rowClasses.join(' '); - if (job.parent_job_id) { - row.dataset.parentId = job.parent_job_id; - } - - const expanderHtml = job.task_type === 'unzip' ? '' : ''; - - row.innerHTML = ` - ${checkboxHtml} - ${expanderHtml}${escapedFilename} - ${fileSizeHtml} - ${taskTypeLabel} - ${formattedDate} - ${statusHtml} - ${actionHtml} - `; - } - - // --- App Initialization and Auth Check --- function initializeApp() { if (appContainer) appContainer.style.display = 'block'; if (loginContainer) loginContainer.style.display = 'none'; + // Setup event listeners startConversionBtn.addEventListener('click', () => handleTaskRequest('conversion')); startOcrBtn.addEventListener('click', () => handleTaskRequest('ocr')); startTranscriptionBtn.addEventListener('click', () => handleTaskRequest('transcription')); startTtsBtn.addEventListener('click', () => handleTaskRequest('tts')); mainFileInput.addEventListener('change', () => updateFileName(mainFileInput, mainFileName)); - - jobListBody.addEventListener('click', (event) => { - if (event.target.classList.contains('cancel-button')) { - const jobId = event.target.dataset.jobId; - handleCancelJob(jobId); - return; + downloadSelectedBtn.addEventListener('click', handleBatchDownload); + selectAllJobsCheckbox.addEventListener('change', handleSelectionChange); + jobListBody.addEventListener('change', e => e.target.classList.contains('job-checkbox') && handleSelectionChange()); + jobListBody.addEventListener('click', e => { + if (e.target.classList.contains('cancel-button')) { + e.preventDefault(); + handleCancelJob(e.target.dataset.jobId); } - // Event delegation for collapsible rows - const parentRow = event.target.closest('tr.parent-job'); - if (parentRow) { - const parentId = parentRow.id.replace('job-', ''); + const parentRow = e.target.closest('tr.parent-job'); + if (parentRow && !e.target.classList.contains('cancel-button') && !e.target.classList.contains('download-button')) { parentRow.classList.toggle('sub-jobs-visible'); const areVisible = parentRow.classList.contains('sub-jobs-visible'); - - // Toggle visibility of all child job rows - const subJobs = jobListBody.querySelectorAll(`tr.sub-job[data-parent-id="${parentId}"]`); - subJobs.forEach(subJob => { - // Use classes instead of direct style manipulation for robustness - if (areVisible) { - subJob.classList.add('is-visible'); - } else { - subJob.classList.remove('is-visible'); - } - }); + jobListBody.querySelectorAll(`tr.sub-job[data-parent-id="${parentRow.id.replace('job-', '')}"]`) + .forEach(subJob => { + subJob.style.display = areVisible ? 'table-row' : 'none'; + }); } }); - jobListBody.addEventListener('change', (event) => { - if (event.target.classList.contains('job-checkbox')) { - handleSelectionChange(); - } - }); + // Dialog listeners + dialogConvertBtn.addEventListener('click', () => { dialogInitialView.style.display = 'none'; dialogConvertView.style.display = 'block'; }); + dialogTtsBtn.addEventListener('click', () => { dialogInitialView.style.display = 'none'; dialogTtsView.style.display = 'block'; }); + dialogBackBtn.addEventListener('click', () => { dialogInitialView.style.display = 'grid'; dialogConvertView.style.display = 'none'; }); + dialogBackTtsBtn.addEventListener('click', () => { dialogInitialView.style.display = 'grid'; dialogTtsView.style.display = 'none'; }); + dialogStartConversionBtn.addEventListener('click', () => handleDialogAction('conversion')); + dialogStartTtsBtn.addEventListener('click', () => handleDialogAction('tts')); + dialogOcrBtn.addEventListener('click', () => handleDialogAction('ocr')); + dialogTranscribeBtn.addEventListener('click', () => handleDialogAction('transcription')); + dialogCancelBtn.addEventListener('click', closeActionDialog); - selectAllJobsCheckbox.addEventListener('change', () => { - const isChecked = selectAllJobsCheckbox.checked; - jobListBody.querySelectorAll('.job-checkbox').forEach(cb => { - cb.checked = isChecked; - }); - handleSelectionChange(); - }); - downloadSelectedBtn.addEventListener('click', handleBatchDownload); - - // Load initial data and setup UI components + // Initialize UI initializeSelectors(); loadInitialJobs(); setupDragAndDropListeners(); } function showLoginView() { - if (appContainer) appContainer.style.display = 'none'; - if (loginContainer) loginContainer.style.display = 'flex'; - if (loginButton) { - loginButton.addEventListener('click', () => { - window.location.href = apiUrl('/login'); - }); - } + if (appContainer) appContainer.style.display = 'block'; + if (loginContainer) loginContainer.style.display = 'none'; + if (loginButton) loginButton.addEventListener('click', () => { window.location.href = apiUrl('/login'); }); } // --- Entry Point --- @@ -786,4 +560,4 @@ document.addEventListener('DOMContentLoaded', () => { } else { showLoginView(); } -}); \ No newline at end of file +}) \ No newline at end of file diff --git a/static/js/script.old b/static/js/script.old old mode 100644 new mode 100755 diff --git a/static/js/settings.js b/static/js/settings.js old mode 100644 new mode 100755 index 4ce6e1c..68d3176 --- a/static/js/settings.js +++ b/static/js/settings.js @@ -5,47 +5,51 @@ document.addEventListener('DOMContentLoaded', () => { const clearHistoryBtn = document.getElementById('clear-history-btn'); const deleteFilesBtn = document.getElementById('delete-files-btn'); - // --- Save Settings --- + // --- Save Settings --- settingsForm.addEventListener('submit', async (event) => { event.preventDefault(); saveStatus.textContent = 'Saving...'; saveStatus.classList.remove('success', 'error'); - const formData = new FormData(settingsForm); const settingsObject = {}; + const elements = Array.from(settingsForm.elements); - // Convert FormData to a nested object - formData.forEach((value, key) => { - // Handle checkboxes that might not be submitted if unchecked - if (key.includes('ocr_settings')) { - const checkbox = document.querySelector(`[name="${key}"]`); - if (checkbox && checkbox.type === 'checkbox') { - value = checkbox.checked; - } + for (const el of elements) { + if (!el.name || el.type === 'submit') continue; // Skip elements without a name and submit buttons + + let value; + const keys = el.name.split('.'); + + // Determine value based on element type + if (el.type === 'checkbox') { + value = el.checked; + } else if (el.tagName === 'TEXTAREA') { + // Convert comma-separated text into an array of strings + value = el.value.split(',') + .map(item => item.trim()) + .filter(item => item); // Remove empty strings from the list + } else if (el.type === 'number') { + value = parseFloat(el.value); + if (isNaN(value)) { + value = null; // Represent empty number fields as null + } + } else { + value = el.value; } - const keys = key.split('.'); + // Build nested object from dot-notation name let current = settingsObject; keys.forEach((k, index) => { if (index === keys.length - 1) { current[k] = value; } else { - current[k] = current[k] || {}; + if (!current[k]) { + current[k] = {}; + } current = current[k]; } }); - }); - - // Ensure unchecked OCR boxes are sent as false - const ocrCheckboxes = settingsForm.querySelectorAll('input[type="checkbox"][name^="ocr_settings"]'); - ocrCheckboxes.forEach(cb => { - const keys = cb.name.split('.'); - if (!formData.has(cb.name)) { - // this is a bit of a hack but gets the job done for this specific form - settingsObject[keys[0]][keys[1]][keys[2]] = false; - } - }); - + } try { const response = await fetch('/settings/save', { @@ -74,7 +78,7 @@ document.addEventListener('DOMContentLoaded', () => { } }); - // --- Clear History --- + // --- Clear History --- clearHistoryBtn.addEventListener('click', async () => { if (!confirm('ARE YOU SURE?\n\nThis will permanently delete all job history records from the database.')) { return; @@ -90,7 +94,7 @@ document.addEventListener('DOMContentLoaded', () => { } }); - // --- Delete Files --- + // --- Delete Files --- deleteFilesBtn.addEventListener('click', async () => { if (!confirm('ARE YOU SURE?\n\nThis will permanently delete all files in the "processed" folder.')) { return; @@ -105,4 +109,4 @@ document.addEventListener('DOMContentLoaded', () => { console.error(error); } }); -}); \ No newline at end of file +}); diff --git a/supervisor.conf b/supervisor.conf old mode 100644 new mode 100755 diff --git a/swappy-20250920_155526.png b/swappy-20250920_155526.png old mode 100644 new mode 100755 diff --git a/templates/index.html b/templates/index.html old mode 100644 new mode 100755 diff --git a/templates/index.old b/templates/index.old old mode 100644 new mode 100755 diff --git a/templates/settings.html b/templates/settings.html old mode 100644 new mode 100755 index 6c76277..f902d79 --- a/templates/settings.html +++ b/templates/settings.html @@ -6,9 +6,7 @@ Settings - File Wizard - - - +
@@ -21,14 +19,44 @@
- -
-

General Settings

 -
- - -
-
+
+
+

General Settings

 +
+ +

The public-facing base URL of the application (e.g., https://files.example.com). Used for generating absolute URLs in webhooks.

+ +
+
+ + +
+
+ +

A comma-separated list of file extensions (e.g., .pdf, .docx, .png). If empty, all files are allowed.

+ +
+
+ +
+

Performance Tuning

 +
+ +

Maximum number of AI models (e.g., Piper TTS) that can run in parallel. Helps prevent CPU/GPU overload.

+ +
+
+ +

Time in seconds before an unused Whisper model is unloaded from memory.

+ +
+
+ +

How often to check for inactive models to unload.

+ +
+
+

OCR (ocrmypdf)

 @@ -48,6 +76,7 @@

Transcription (Whisper)

 +

Device settings (CPU/GPU) are configured via environment variables (see documentation).

+
+
+ + +
+
+ + +
+
+ +

Comma-separated list of email addresses for users who should have admin rights.

+ +
+
+ +
+

Webhooks

 +

Allow programmatic access and job status callbacks.

+
+ + +
+
+ + +
+
+ +

Comma-separated list of URLs or domain prefixes that are allowed for callbacks (e.g., https://n8n.example.com).

+ +
+
+ +

If set, this token will be sent in the `Authorization` header for all callback requests.

+ +
+
+ +
+

TTS (Piper)

 +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+

Conversion Tools