diff --git a/Cargo.lock b/Cargo.lock index d5cfac32..200f5a35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1773,7 +1773,7 @@ dependencies = [ [[package]] name = "nzb-core" -version = "0.2.18" +version = "0.2.19" dependencies = [ "anyhow", "chrono", @@ -1894,7 +1894,7 @@ dependencies = [ [[package]] name = "nzb-web" -version = "0.4.23" +version = "0.4.24" dependencies = [ "anyhow", "arc-swap", diff --git a/Cargo.toml b/Cargo.toml index 419599e3..9df3672d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,9 +88,9 @@ feed-rs = "2" unicode-normalization = "0.1" # Shared NZB crates -nzb-web = { version = "0.4.23", path = "crates/nzb-web", features = ["groups-db"] } +nzb-web = { version = "0.4.24", path = "crates/nzb-web", features = ["groups-db"] } nzb-nntp = { version = "0.2.24", path = "crates/nzb-nntp" } -nzb-core = { version = "0.2.18", path = "crates/nzb-core", features = ["groups-db"] } +nzb-core = { version = "0.2.19", path = "crates/nzb-core", features = ["groups-db"] } nzb-decode = { version = "0.1.5", path = "crates/nzb-decode" } nzb-news = { version = "0.1.14", path = "crates/nzb-news" } nzb-dispatch = { version = "0.2.8", path = "crates/nzb-dispatch" } diff --git a/README.md b/README.md index a82ed842..b5353317 100644 --- a/README.md +++ b/README.md @@ -205,14 +205,26 @@ curl -X POST http://localhost:9090/api/queue/add-url \ -H "Content-Type: application/json" \ -d '{"url": "https://example.com/file.nzb", "category": "movies"}' -# Upload NZB file +# Upload NZB file (optional fields: category/cat, priority 0-3, +# name/nzbname; each may also be passed as a query parameter, and a form +# field wins over the query) curl -X POST http://localhost:9090/api/queue/add \ - -F "file=@/path/to/file.nzb" -F "category=tv" + -F "file=@/path/to/file.nzb" -F "category=tv" -F "priority=2" + +# Upload an NZB with an archive password (multipart `password` field; +# `add-url` takes a JSON `password`) +curl -X POST http://localhost:9090/api/queue/add \ + -F "file=@/path/to/file.nzb" -F "password=secret" # Check status curl http://localhost:9090/api/status ``` +Job names follow SABnzbd's inline-password convention on both APIs: a name +or NZB file name of the form `Name{{password}}` or `Name/password` sets the +job password and is queued as `Name`. An explicit `password` wins over the +inline one, which wins over a password embedded in the NZB. + The *arr-compatible API is served at `/sabnzbd/api`. --- diff --git a/apps/rustnzb/config.example.toml b/apps/rustnzb/config.example.toml index 1c2f4791..ad665b50 100644 --- a/apps/rustnzb/config.example.toml +++ b/apps/rustnzb/config.example.toml @@ -6,20 +6,33 @@ incomplete_dir = "/downloads/incomplete" complete_dir = "/downloads/complete" data_dir = "/data" speed_limit_bps = 0 # 0 = unlimited -cache_size = 524288000 # 500 MB +cache_size = 524288000 # 500 MB; the API accepts 16 MiB to 64 GiB +# Log level: trace, debug, info, warn, error, or off. A full filter such as +# "info,nzb_nntp=debug" is also accepted. Precedence, highest first: +# RUST_LOG env (full filter override) > --log-level CLI flag > +# RUSTNZB_LOG_LEVEL env > this setting > "info". +# An invalid value logs a warning and falls back to "info". Applied at start. log_level = "info" +# max_active_downloads = 1 # 0 = unlimited; the API accepts at most 100 # log_file = "data/rustnzb.log" # history_retention = 100 # Number of NZBs to keep in history (omit or 0 = keep all) +max_active_downloads = 1 # NZBs downloading simultaneously (0 = unlimited) # auto_sort_remaining_pct = false # Keep queued items ordered by remaining work # rss_downloaded_item_expiry_days = 30 # Remove downloaded RSS records after this many days # Independent post-processing jobs overlap downloads. Separate repair and # extraction gates allow those stages to overlap safely across different jobs. -# These limits are applied at process start. +# These limits are applied at process start. The API accepts 0..=64 per +# limit (0 is raised to 1). max_post_processing_jobs = 2 max_repair_workers = 1 max_extract_workers = 1 direct_unpack = true # Extract RAR volumes while downloading when unrar is available -max_nested_archive_depth = 5 # 0=outer archive only; protects against unbounded nesting +max_nested_archive_depth = 5 # 0=outer archive only; protects against unbounded nesting (API: 0-10) +# The download-health settings below are applied at process start. +# abort_hopeless = true # edited via /api/config/disk-guards +# early_failure_check = true # abort dead NZBs after the first articles +# required_completion_pct = 100.2 # 100.0-200.0; >100 reserves a PAR2 repair margin +# article_timeout_secs = 30 # 0 = no timeout; the API accepts at most 3600 # Post-processing hooks run without a shell, with a bounded timeout and output capture. # scripts_dir = "/data/scripts" # script_success = "on-success.sh" diff --git a/apps/rustnzb/frontend/src/app/app.spec.ts b/apps/rustnzb/frontend/src/app/app.spec.ts index 6cb50f48..591230f0 100644 --- a/apps/rustnzb/frontend/src/app/app.spec.ts +++ b/apps/rustnzb/frontend/src/app/app.spec.ts @@ -23,6 +23,7 @@ function makeApp(postResult = new Subject()) { navigate: vi.fn(() => Promise.resolve(true)), }; const pauseState = new PauseStateService(); + const snack = { open: vi.fn() }; const app = new App( api as never, auth as never, @@ -31,8 +32,9 @@ function makeApp(postResult = new Subject()) { {} as never, {} as never, pauseState, + snack as never, ); - return { app, api, pauseState, postResult }; + return { app, api, pauseState, postResult, snack }; } describe('App global pause control', () => { @@ -46,13 +48,16 @@ describe('App global pause control', () => { }); it('rolls back the shared state if the global request fails', () => { - const { app, pauseState, postResult } = makeApp(); + const { app, pauseState, postResult, snack } = makeApp(); vi.spyOn(app, 'pollStatus').mockImplementation(() => {}); app.togglePause(); postResult.error(new Error('request failed')); expect(pauseState.paused()).toBe(false); + expect(snack.open).toHaveBeenCalledWith('Failed to pause downloads', 'Close', { + duration: 5000, + }); }); }); diff --git a/apps/rustnzb/frontend/src/app/app.ts b/apps/rustnzb/frontend/src/app/app.ts index e5a787e6..b4558cba 100644 --- a/apps/rustnzb/frontend/src/app/app.ts +++ b/apps/rustnzb/frontend/src/app/app.ts @@ -21,6 +21,8 @@ import { WidthModeService } from './core/services/width-mode.service'; import { PauseStateService } from './core/services/pause-state.service'; import { ThemeService } from './core/services/theme.service'; import { IconComponent } from './shared/icon.component'; +import { MatSnackBar } from '@angular/material/snack-bar'; +import { showHttpError } from './core/http/http-error'; export function isDemoPath(pathname: string): boolean { return pathname === '/demo' || pathname.startsWith('/demo/'); @@ -451,6 +453,7 @@ export class App implements OnInit, OnDestroy { public widthMode: WidthModeService, readonly theme: ThemeService, pauseState: PauseStateService, + private snack: MatSnackBar, ) { this.paused = pauseState.paused; this.authenticated = authService.authenticated; @@ -509,9 +512,14 @@ export class App implements OnInit, OnDestroy { this.paused.set(!wasPaused); this.api.post(action).subscribe({ next: () => this.pollStatus(), - error: () => { + error: (err) => { this.paused.set(wasPaused); this.pollStatus(); + showHttpError( + this.snack, + err, + wasPaused ? 'Failed to resume downloads' : 'Failed to pause downloads', + ); }, }); this.pauseMenuOpen = false; @@ -524,7 +532,10 @@ export class App implements OnInit, OnDestroy { } pauseFor(secs: number): void { - this.api.post(`/queue/pause-for?duration_secs=${secs}`).subscribe(() => this.pollStatus()); + this.api.post(`/queue/pause-for?duration_secs=${secs}`).subscribe({ + next: () => this.pollStatus(), + error: (err) => showHttpError(this.snack, err, 'Failed to pause downloads'), + }); this.pauseMenuOpen = false; } diff --git a/apps/rustnzb/frontend/src/app/core/http/http-error.spec.ts b/apps/rustnzb/frontend/src/app/core/http/http-error.spec.ts new file mode 100644 index 00000000..d8163f80 --- /dev/null +++ b/apps/rustnzb/frontend/src/app/core/http/http-error.spec.ts @@ -0,0 +1,82 @@ +import '@angular/compiler'; + +import { HttpErrorResponse } from '@angular/common/http'; +import { describe, expect, it, vi } from 'vitest'; + +import { httpErrorDetail, httpErrorMessage, isAuthHandledError, showHttpError } from './http-error'; + +function httpError(status: number, error: unknown): HttpErrorResponse { + return new HttpErrorResponse({ status, error, url: '/api/x' }); +} + +describe('httpErrorMessage', () => { + it('prefers the central human_readable field', () => { + const err = httpError(400, { + error_kind: 'bad_request', + human_readable: 'Feed URL rejected: URL targets a private/reserved address', + status: 400, + }); + expect(httpErrorMessage(err, 'Failed to save feed')).toBe( + 'Failed to save feed: Feed URL rejected: URL targets a private/reserved address', + ); + }); + + it('falls back to the legacy error and message fields', () => { + expect(httpErrorMessage(httpError(400, { status: false, error: 'No job ID' }), 'Failed')).toBe( + 'Failed: No job ID', + ); + expect(httpErrorMessage(httpError(422, { message: 'Bad NZB' }), 'Failed')).toBe( + 'Failed: Bad NZB', + ); + }); + + it('parses JSON bodies delivered as text and accepts short plain-text bodies', () => { + expect( + httpErrorMessage(httpError(400, '{"human_readable":"Invalid regex"}'), 'Failed to save rule'), + ).toBe('Failed to save rule: Invalid regex'); + expect(httpErrorMessage(httpError(415, 'Expected JSON content type'), 'Failed')).toBe( + 'Failed: Expected JSON content type', + ); + }); + + it('uses the fallback when there is no usable server message', () => { + expect(httpErrorMessage(httpError(500, null), 'Failed to save')).toBe('Failed to save'); + expect( + httpErrorMessage(httpError(502, 'Bad gateway'), 'Failed'), + ).toBe('Failed'); + expect(httpErrorMessage(httpError(400, { human_readable: ' ' }), 'Failed')).toBe('Failed'); + expect(httpErrorMessage(new Error('boom'), 'Failed')).toBe('Failed'); + expect(httpErrorMessage(undefined, 'Failed')).toBe('Failed'); + }); + + it('does not repeat the fallback when the server message already is it', () => { + expect(httpErrorMessage(httpError(400, { error: 'Failed' }), 'Failed')).toBe('Failed'); + }); +}); + +describe('httpErrorDetail', () => { + it('returns only the server text, or null', () => { + expect(httpErrorDetail(httpError(400, { human_readable: 'Bad', error: 'ignored' }))).toBe( + 'Bad', + ); + expect(httpErrorDetail(httpError(0, null))).toBeNull(); + expect(httpErrorDetail(new Error('client'))).toBeNull(); + }); +}); + +describe('showHttpError', () => { + it('opens a snackbar with the extracted message', () => { + const snack = { open: vi.fn() }; + showHttpError(snack, httpError(400, { human_readable: 'Bad' }), 'Failed to save'); + expect(snack.open).toHaveBeenCalledWith('Failed to save: Bad', 'Close', { duration: 5000 }); + }); + + it('stays silent on 401s, which the auth interceptor already handles', () => { + const snack = { open: vi.fn() }; + const err = httpError(401, { human_readable: 'Unauthorized' }); + expect(isAuthHandledError(err)).toBe(true); + showHttpError(snack, err, 'Failed to save'); + expect(snack.open).not.toHaveBeenCalled(); + expect(isAuthHandledError(httpError(403, null))).toBe(false); + }); +}); diff --git a/apps/rustnzb/frontend/src/app/core/http/http-error.ts b/apps/rustnzb/frontend/src/app/core/http/http-error.ts new file mode 100644 index 00000000..2dd6e1e6 --- /dev/null +++ b/apps/rustnzb/frontend/src/app/core/http/http-error.ts @@ -0,0 +1,80 @@ +import type { HttpErrorResponse } from '@angular/common/http'; + +/** Anything with MatSnackBar's `open` signature (kept narrow for tests). */ +export interface SnackOpener { + open(message: string, action?: string, config?: { duration?: number }): unknown; +} + +const MAX_TEXT_BODY = 300; + +function nonEmpty(value: unknown): string | null { + return typeof value === 'string' && value.trim() ? value.trim() : null; +} + +function fromBody(body: unknown): string | null { + if (typeof body === 'string') { + const text = body.trim(); + if (!text) return null; + if (text.startsWith('{')) { + try { + return fromBody(JSON.parse(text)); + } catch { + // Not JSON after all; treat it as plain text below. + } + } + // Short plain-text bodies (e.g. axum extractor rejections) are useful; + // HTML error pages from a proxy are not. + if (text.startsWith('<') || text.length > MAX_TEXT_BODY) return null; + return text; + } + if (body && typeof body === 'object') { + const b = body as Record; + // Central API error shape first, then the legacy `{status:false, error}` + // shape, then older `{message}` bodies. + return nonEmpty(b['human_readable']) ?? nonEmpty(b['error']) ?? nonEmpty(b['message']); + } + return null; +} + +/** + * The server's own explanation of a failed request, or `null` when the + * response carries nothing usable (network error, HTML proxy page, ...). + */ +export function httpErrorDetail(err: unknown): string | null { + // Duck-typed rather than `instanceof HttpErrorResponse` so errors that were + // re-thrown or built by hand with the same `{status, error}` shape work too. + if (!err || typeof err !== 'object' || !('error' in err)) return null; + return fromBody((err as HttpErrorResponse).error); +} + +/** + * Turn a failed request into a user-facing message: the server's own + * explanation (`human_readable`, falling back to `error`/`message`) prefixed + * with what the user was trying to do, or just `fallback` when the response + * carries nothing usable. + */ +export function httpErrorMessage(err: unknown, fallback: string): string { + const detail = httpErrorDetail(err); + if (!detail || detail === fallback) return fallback; + return `${fallback}: ${detail}`; +} + +/** + * A 401 reaching a component means the auth interceptor already tried to + * refresh the session and has redirected to the login page; a toast on top + * of that redirect is noise. + */ +export function isAuthHandledError(err: unknown): boolean { + return !!err && typeof err === 'object' && (err as HttpErrorResponse).status === 401; +} + +/** Show a failed request as a toast, unless the auth flow is handling it. */ +export function showHttpError( + snack: SnackOpener, + err: unknown, + fallback: string, + duration = 5000, +): void { + if (isAuthHandledError(err)) return; + snack.open(httpErrorMessage(err, fallback), 'Close', { duration }); +} diff --git a/apps/rustnzb/frontend/src/app/core/models/queue.model.ts b/apps/rustnzb/frontend/src/app/core/models/queue.model.ts index bd8a1456..73bd5cda 100644 --- a/apps/rustnzb/frontend/src/app/core/models/queue.model.ts +++ b/apps/rustnzb/frontend/src/app/core/models/queue.model.ts @@ -6,6 +6,9 @@ export interface ServerArticleStats { bytes_downloaded: number; } +/** Why a job or the whole queue is paused; `disk_space` lifts automatically. */ +export type PauseReason = 'manual' | 'global' | 'disk_space' | 'server'; + export interface NzbJob { id: string; name: string; @@ -24,6 +27,7 @@ export interface NzbJob { speed_bps: number; error_message: string | null; server_stats: ServerArticleStats[]; + pause_reason?: PauseReason | null; } export interface QueueResponse { @@ -31,6 +35,7 @@ export interface QueueResponse { total: number; speed_bps: number; paused: boolean; + pause_reason?: PauseReason | null; } export interface StatusResponse { @@ -43,6 +48,7 @@ export interface StatusResponse { disk_space_total: number; min_free_space_bytes: number; paused: boolean; + pause_reason?: PauseReason | null; pause_remaining_secs: number | null; webdav_available: boolean; webdav_enabled: boolean; diff --git a/apps/rustnzb/frontend/src/app/features/auth/login.component.ts b/apps/rustnzb/frontend/src/app/features/auth/login.component.ts index d9a3f7ab..0d1e1a67 100644 --- a/apps/rustnzb/frontend/src/app/features/auth/login.component.ts +++ b/apps/rustnzb/frontend/src/app/features/auth/login.component.ts @@ -3,6 +3,7 @@ import { CommonModule } from '@angular/common'; import { FormsModule } from '@angular/forms'; import { Router } from '@angular/router'; import { AuthService } from '../../core/services/auth.service'; +import { httpErrorDetail } from '../../core/http/http-error'; @Component({ selector: 'app-login', @@ -210,10 +211,8 @@ export class LoginComponent implements OnInit { this.errorMessage.set('Invalid username or password.'); } else if (err.status === 409) { this.errorMessage.set('An account already exists. Please sign in instead.'); - } else if (err.error?.message) { - this.errorMessage.set(err.error.message); } else { - this.errorMessage.set('An error occurred. Please try again.'); + this.errorMessage.set(httpErrorDetail(err) ?? 'An error occurred. Please try again.'); } }, }); diff --git a/apps/rustnzb/frontend/src/app/features/groups/group-browser-dialog.component.ts b/apps/rustnzb/frontend/src/app/features/groups/group-browser-dialog.component.ts index 78d6c38b..47d25741 100644 --- a/apps/rustnzb/frontend/src/app/features/groups/group-browser-dialog.component.ts +++ b/apps/rustnzb/frontend/src/app/features/groups/group-browser-dialog.component.ts @@ -6,10 +6,10 @@ import { MatButtonModule } from '@angular/material/button'; import { MatIconModule } from '@angular/material/icon'; import { MatProgressBarModule } from '@angular/material/progress-bar'; import { MatSnackBar, MatSnackBarModule } from '@angular/material/snack-bar'; -import { HttpErrorResponse } from '@angular/common/http'; import { GroupService } from '../../core/services/group.service'; import { GroupRow } from '../../core/models/group.model'; import { IconComponent } from '../../shared/icon.component'; +import { showHttpError } from '../../core/http/http-error'; @Component({ selector: 'app-group-browser-dialog', @@ -100,12 +100,9 @@ export class GroupBrowserDialogComponent implements OnInit { this.refreshing.set(true); this.svc.refresh().subscribe({ next: r => { this.refreshing.set(false); this.snack.open(r.message, 'Close', { duration: 3000 }); this.loadGroups(); }, - error: (e: HttpErrorResponse) => { + error: (e) => { this.refreshing.set(false); - const msg = e.status === 400 - ? (e.error?.human_readable || 'No servers configured — add one in Settings first.') - : 'Refresh failed'; - this.snack.open(msg, 'Close', { duration: 5000 }); + showHttpError(this.snack, e, 'Refresh failed'); }, }); } @@ -132,11 +129,11 @@ export class GroupBrowserDialogComponent implements OnInit { ); clearPending(); }, - error: () => { - this.snack.open( + error: (err) => { + showHttpError( + this.snack, + err, `Failed to ${wasSubscribed ? 'unsubscribe from' : 'subscribe to'} ${g.name}`, - 'Close', - { duration: 4000 }, ); clearPending(); }, diff --git a/apps/rustnzb/frontend/src/app/features/groups/groups-view.component.ts b/apps/rustnzb/frontend/src/app/features/groups/groups-view.component.ts index df5ca485..0b69b546 100644 --- a/apps/rustnzb/frontend/src/app/features/groups/groups-view.component.ts +++ b/apps/rustnzb/frontend/src/app/features/groups/groups-view.component.ts @@ -7,6 +7,7 @@ import { GroupService } from '../../core/services/group.service'; import { GroupRow, HeaderRow } from '../../core/models/group.model'; import { GroupBrowserDialogComponent } from './group-browser-dialog.component'; import { IconComponent } from '../../shared/icon.component'; +import { showHttpError } from '../../core/http/http-error'; @Component({ selector: 'app-groups-view', @@ -493,7 +494,10 @@ export class GroupsViewComponent implements OnInit { this.fetching.set(true); this.svc.fetchHeaders(g.id).subscribe({ next: () => this.snack.open('Fetching headers…', 'Close', { duration: 2000 }), - error: () => this.fetching.set(false), + error: (err) => { + this.fetching.set(false); + showHttpError(this.snack, err, 'Failed to fetch headers'); + }, }); const poll = setInterval(() => { this.loadHeaders(); @@ -519,10 +523,13 @@ export class GroupsViewComponent implements OnInit { markAllRead(): void { const g = this.selectedGroup(); if (!g) return; - this.svc.markAllRead(g.id).subscribe(() => { - this.loadHeaders(); - this.loadGroups(); - this.snack.open('All marked read', 'Close', { duration: 2000 }); + this.svc.markAllRead(g.id).subscribe({ + next: () => { + this.loadHeaders(); + this.loadGroups(); + this.snack.open('All marked read', 'Close', { duration: 2000 }); + }, + error: (err) => showHttpError(this.snack, err, 'Failed to mark all read'), }); } @@ -567,7 +574,7 @@ export class GroupsViewComponent implements OnInit { this.snack.open(r.message, 'Close', { duration: 3000 }); this.selectedIds.set([]); }, - error: () => this.snack.open('Download failed', 'Close', { duration: 5000 }), + error: (err) => showHttpError(this.snack, err, 'Download failed'), }); } diff --git a/apps/rustnzb/frontend/src/app/features/history/history-view.component.ts b/apps/rustnzb/frontend/src/app/features/history/history-view.component.ts index db64d0dd..ea8a9a53 100644 --- a/apps/rustnzb/frontend/src/app/features/history/history-view.component.ts +++ b/apps/rustnzb/frontend/src/app/features/history/history-view.component.ts @@ -6,6 +6,7 @@ import { ApiService } from '../../core/services/api.service'; import { HistoryEntry, StatusResponse } from '../../core/models/queue.model'; import { ConfirmService } from '../../shared/confirm.service'; import { IconComponent } from '../../shared/icon.component'; +import { showHttpError } from '../../core/http/http-error'; type StatusFilter = 'all' | 'completed' | 'failed'; type TimeFilter = '7d' | '30d' | 'all'; @@ -407,21 +408,27 @@ export class HistoryViewComponent implements OnInit, OnDestroy { }); retry(id: string): void { - this.api.post(`/history/${id}/retry`).subscribe(() => { - this.load(); - this.snack.open('Retrying…', 'Close', { duration: 2000 }); + this.api.post(`/history/${id}/retry`).subscribe({ + next: () => { + this.load(); + this.snack.open('Retrying…', 'Close', { duration: 2000 }); + }, + error: (err) => showHttpError(this.snack, err, 'Failed to retry job'), }); } addToMedia(id: string): void { this.api.post(`/dav/add?id=${id}`).subscribe({ next: () => this.snack.open('Queued for Media Library', 'Close', { duration: 3000 }), - error: () => this.snack.open('Failed to add to Media Library', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to add to Media Library'), }); } remove(id: string): void { - this.api.delete(`/history/${id}`).subscribe(() => this.load()); + this.api.delete(`/history/${id}`).subscribe({ + next: () => this.load(), + error: (err) => showHttpError(this.snack, err, 'Failed to delete history entry'), + }); } clearAll(): void { @@ -434,9 +441,12 @@ export class HistoryViewComponent implements OnInit, OnDestroy { }) .subscribe((ok) => { if (!ok) return; - this.api.delete('/history').subscribe(() => { - this.load(); - this.snack.open('History cleared', 'Close', { duration: 2000 }); + this.api.delete('/history').subscribe({ + next: () => { + this.load(); + this.snack.open('History cleared', 'Close', { duration: 2000 }); + }, + error: (err) => showHttpError(this.snack, err, 'Failed to clear history'), }); }); } diff --git a/apps/rustnzb/frontend/src/app/features/queue/queue-view.component.spec.ts b/apps/rustnzb/frontend/src/app/features/queue/queue-view.component.spec.ts index e30dca81..8d5e222f 100644 --- a/apps/rustnzb/frontend/src/app/features/queue/queue-view.component.spec.ts +++ b/apps/rustnzb/frontend/src/app/features/queue/queue-view.component.spec.ts @@ -1,5 +1,6 @@ import '@angular/compiler'; +import { HttpErrorResponse } from '@angular/common/http'; import { Observable, of, Subject, throwError } from 'rxjs'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { convertToParamMap, ParamMap } from '@angular/router'; @@ -174,6 +175,48 @@ describe('QueueViewComponent', () => { expect(component.displayConnectionCount('backup', 2_000)).toEqual({ count: 3, recent: false }); }); + it('labels a job held by the disk-space guard', () => { + const { component } = makeComponent(); + + expect(component.statusLabel(makeJob({ status: 'paused', pause_reason: 'disk_space' }))).toBe( + 'paused · low disk', + ); + expect(component.statusLabel(makeJob({ status: 'paused', pause_reason: 'manual' }))).toBe( + 'paused', + ); + expect(component.statusLabel(makeJob({ status: 'downloading' }))).toBe('downloading'); + }); + + it('shows the disk-space reason for a global pause applied by the guard', () => { + const { component } = makeComponent({ + get: vi.fn((url: string) => + url === '/queue' + ? of({ jobs: [], total: 0, speed_bps: 0, paused: true, pause_reason: 'disk_space' }) + : of({}), + ), + }); + + component.loadQueue(); + + expect(component.pausedLabel()).toBe('Paused · low disk space'); + // A queued job caught by the global hold before the next refresh. + expect(component.statusLabel(makeJob({ status: 'queued' }))).toBe('paused · low disk'); + }); + + it('keeps the plain paused label for a user pause', () => { + const { component } = makeComponent({ + get: vi.fn((url: string) => + url === '/queue' + ? of({ jobs: [], total: 0, speed_bps: 0, paused: true, pause_reason: 'global' }) + : of({}), + ), + }); + + component.loadQueue(); + + expect(component.pausedLabel()).toBe('Paused'); + }); + it('ignores a duplicate row action while the first request is pending', () => { const action$ = new Subject(); const { component, api } = makeComponent({ @@ -236,14 +279,26 @@ describe('QueueViewComponent', () => { it('reloads queue and shows feedback after a failed row action', () => { const { component, snackBar } = makeComponent({ - delete: vi.fn(() => throwError(() => new Error('Delete failed'))), + delete: vi.fn(() => + throwError( + () => + new HttpErrorResponse({ + status: 409, + error: { error_kind: 'conflict', human_readable: 'Job is post-processing', status: 409 }, + }), + ), + ), }); const loadQueue = vi.spyOn(component, 'loadQueue').mockImplementation(() => {}); component.deleteJob(makeJob({ id: 'job-1' })); expect(loadQueue).toHaveBeenCalledTimes(1); - expect(snackBar.open).toHaveBeenCalledWith('Delete failed', 'Close', { duration: 4000 }); + expect(snackBar.open).toHaveBeenCalledWith( + 'Failed to remove job: Job is post-processing', + 'Close', + { duration: 5000 }, + ); }); it('reorders jobs before the target row', () => { @@ -318,4 +373,38 @@ describe('QueueViewComponent', () => { expect(component.historyCollapsed()).toBe(false); expect(localStorage.getItem(component.HISTORY_KEY)).toBe('false'); }); + + it('reports bulk action failures once with the server message', () => { + const notFound = new HttpErrorResponse({ + status: 404, + error: { error_kind: 'job_not_found', human_readable: 'Job not found: job-2', status: 404 }, + }); + const { component, snackBar } = makeComponent({ + post: vi.fn((path: string) => (path.includes('job-2') ? throwError(() => notFound) : of({}))), + }); + const loadQueue = vi.spyOn(component, 'loadQueue').mockImplementation(() => {}); + component.selectedIds.set(new Set(['job-1', 'job-2'])); + + component.bulkPause(); + + expect(snackBar.open).toHaveBeenCalledTimes(1); + expect(snackBar.open).toHaveBeenCalledWith( + 'Failed to pause 1 of 2 job(s): Job not found: job-2', + 'Close', + { duration: 5000 }, + ); + expect(loadQueue).toHaveBeenCalledTimes(1); + expect(component.selectedIds().size).toBe(0); + }); + + it('does not toast a 401 that the auth interceptor is handling', () => { + const { component, snackBar } = makeComponent({ + post: vi.fn(() => throwError(() => new HttpErrorResponse({ status: 401 }))), + }); + vi.spyOn(component, 'loadQueue').mockImplementation(() => {}); + + component.pauseJob('job-1'); + + expect(snackBar.open).not.toHaveBeenCalled(); + }); }); diff --git a/apps/rustnzb/frontend/src/app/features/queue/queue-view.component.ts b/apps/rustnzb/frontend/src/app/features/queue/queue-view.component.ts index 0f2d64ec..1b13ad4a 100644 --- a/apps/rustnzb/frontend/src/app/features/queue/queue-view.component.ts +++ b/apps/rustnzb/frontend/src/app/features/queue/queue-view.component.ts @@ -4,14 +4,15 @@ import { FormsModule } from '@angular/forms'; import { ActivatedRoute, Router, RouterModule } from '@angular/router'; import { HttpClient } from '@angular/common/http'; import { MatSnackBar, MatSnackBarModule } from '@angular/material/snack-bar'; -import { Observable, Subscription, finalize } from 'rxjs'; +import { Observable, Subscription, catchError, finalize, forkJoin, map, of } from 'rxjs'; import { ApiService } from '../../core/services/api.service'; import { AddNzbService } from '../../core/services/add-nzb.service'; import { PauseStateService } from '../../core/services/pause-state.service'; -import { NzbJob, QueueResponse, StatusResponse } from '../../core/models/queue.model'; +import { NzbJob, PauseReason, QueueResponse, StatusResponse } from '../../core/models/queue.model'; import { HistoryViewComponent } from '../history/history-view.component'; import { ConfirmService } from '../../shared/confirm.service'; import { IconComponent } from '../../shared/icon.component'; +import { isAuthHandledError, showHttpError } from '../../core/http/http-error'; interface CategoryConfig { name: string; @@ -66,7 +67,7 @@ interface PipelineStep {
{{ speedValue() }} {{ speedUnit() }}
-
{{ paused() ? 'Paused' : 'Active · limit off' }}
+
{{ paused() ? pausedLabel() : 'Active · limit off' }}
NNTP connections
@@ -429,9 +430,16 @@ interface PipelineStep { {{ job.speed_bps > 0 ? formatSpeed(job.speed_bps) : '—' }} {{ job.speed_bps > 0 ? eta(job) : '—' }} - {{ - displayStatus(effectiveStatus(job.status)) - }} + {{ statusLabel(job) }} Auto-download matches
+ @if (feedError()) { + + }
@@ -154,6 +158,9 @@ interface RuleFormModel {
+ @if (ruleError()) { + + }
@@ -270,6 +277,12 @@ interface RuleFormModel { border-bottom: 1px solid var(--line); } .form-actions { margin-top: 14px; display: flex; gap: 8px; } + .form-error { + margin-top: 14px; + background: color-mix(in srgb, var(--danger) 10%, transparent); + border: 1px solid var(--danger); + border-radius: 6px; padding: 10px 14px; color: var(--danger); font-size: 13px; + } td.dim { color: var(--mute); } .dim { color: var(--mute); } @@ -285,10 +298,12 @@ export class RssViewComponent implements OnInit { feedFormVisible = signal(false); editingFeedName = signal(null); feedForm: FeedFormModel = this.emptyFeedForm(); + feedError = signal(null); ruleFormVisible = signal(false); editingRuleId = signal(null); ruleForm: RuleFormModel = this.emptyRuleForm(); + ruleError = signal(null); // ---- Stat-card derivations ---- enabledFeedCount = computed(() => this.feeds().filter(f => f.enabled).length); @@ -332,6 +347,7 @@ export class RssViewComponent implements OnInit { showAddFeed(): void { this.feedForm = this.emptyFeedForm(); this.editingFeedName.set(null); + this.feedError.set(null); this.feedFormVisible.set(true); } @@ -346,12 +362,14 @@ export class RssViewComponent implements OnInit { auto_download: f.auto_download, }; this.editingFeedName.set(f.name); + this.feedError.set(null); this.feedFormVisible.set(true); } cancelFeedForm(): void { this.feedFormVisible.set(false); this.editingFeedName.set(null); + this.feedError.set(null); } saveFeed(): void { @@ -378,7 +396,10 @@ export class RssViewComponent implements OnInit { this.cancelFeedForm(); this.loadAll(); }, - error: () => this.snack.open('Failed to save feed', 'Close', { duration: 3000 }), + error: (err) => { + if (!isAuthHandledError(err)) this.feedError.set(httpErrorMessage(err, 'Failed to save feed')); + showHttpError(this.snack, err, 'Failed to save feed'); + }, }); } @@ -397,7 +418,7 @@ export class RssViewComponent implements OnInit { this.snack.open('Feed deleted', 'Close', { duration: 2000 }); this.loadAll(); }, - error: () => this.snack.open('Failed to delete feed', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to delete feed'), }); }); } @@ -407,6 +428,7 @@ export class RssViewComponent implements OnInit { showAddRule(): void { this.ruleForm = this.emptyRuleForm(); this.editingRuleId.set(null); + this.ruleError.set(null); this.ruleFormVisible.set(true); } @@ -420,12 +442,14 @@ export class RssViewComponent implements OnInit { feed_names_csv: r.feed_names.join(', '), }; this.editingRuleId.set(r.id); + this.ruleError.set(null); this.ruleFormVisible.set(true); } cancelRuleForm(): void { this.ruleFormVisible.set(false); this.editingRuleId.set(null); + this.ruleError.set(null); } saveRule(): void { @@ -455,7 +479,10 @@ export class RssViewComponent implements OnInit { this.cancelRuleForm(); this.loadAll(); }, - error: () => this.snack.open('Failed to save rule', 'Close', { duration: 3000 }), + error: (err) => { + if (!isAuthHandledError(err)) this.ruleError.set(httpErrorMessage(err, 'Failed to save rule')); + showHttpError(this.snack, err, 'Failed to save rule'); + }, }); } @@ -474,7 +501,7 @@ export class RssViewComponent implements OnInit { this.snack.open('Rule deleted', 'Close', { duration: 2000 }); this.loadAll(); }, - error: () => this.snack.open('Failed to delete rule', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to delete rule'), }); }); } @@ -487,7 +514,7 @@ export class RssViewComponent implements OnInit { this.snack.open('Added to queue', 'Close', { duration: 2000 }); this.loadAll(); }, - error: () => this.snack.open('Download failed', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Download failed'), }); } diff --git a/apps/rustnzb/frontend/src/app/features/settings/settings-view.component.spec.ts b/apps/rustnzb/frontend/src/app/features/settings/settings-view.component.spec.ts index 88a19e89..5ddc4a08 100644 --- a/apps/rustnzb/frontend/src/app/features/settings/settings-view.component.spec.ts +++ b/apps/rustnzb/frontend/src/app/features/settings/settings-view.component.spec.ts @@ -72,7 +72,7 @@ describe('SettingsViewComponent', () => { }); component.speedLimit = 999; component.saveSpeedLimit(); - expect(snack.open).toHaveBeenCalledWith('Failed to save speed limit', 'Close', { duration: 3000 }); + expect(snack.open).toHaveBeenCalledWith('Failed to save speed limit', 'Close', { duration: 5000 }); component.rotateSabApiKey(); expect(confirm.confirm).toHaveBeenCalledWith(expect.objectContaining({ title: 'Generate SABnzbd API key?' })); @@ -80,4 +80,21 @@ describe('SettingsViewComponent', () => { expect(component.sabApiKey).toBe('rotated'); expect(component.showSabApiKey).toBe(true); }); + + it('shows the server explanation when a settings save is rejected', () => { + const { component, snack } = makeComponent({ + put: vi.fn(() => + throwError(() => ({ + status: 400, + error: { error_kind: 'bad_request', human_readable: 'speed_limit_bps must be >= 0', status: 400 }, + })), + ), + }); + component.saveSpeedLimit(); + expect(snack.open).toHaveBeenCalledWith( + 'Failed to save speed limit: speed_limit_bps must be >= 0', + 'Close', + { duration: 5000 }, + ); + }); }); diff --git a/apps/rustnzb/frontend/src/app/features/settings/settings-view.component.ts b/apps/rustnzb/frontend/src/app/features/settings/settings-view.component.ts index 1f80e6d3..119f54d3 100644 --- a/apps/rustnzb/frontend/src/app/features/settings/settings-view.component.ts +++ b/apps/rustnzb/frontend/src/app/features/settings/settings-view.component.ts @@ -8,6 +8,7 @@ import { StatusResponse } from '../../core/models/queue.model'; import { ConfirmService } from '../../shared/confirm.service'; import { IconComponent } from '../../shared/icon.component'; import { AppTheme, ThemeService } from '../../core/services/theme.service'; +import { showHttpError } from '../../core/http/http-error'; interface ServerConfig { id: string; @@ -1524,7 +1525,7 @@ export class SettingsViewComponent implements OnInit { this.cancelServerEdit(); this.loadServers(); }, - error: () => this.snack.open('Failed to update server', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to update server'), }); } else { server.id = ''; @@ -1534,7 +1535,7 @@ export class SettingsViewComponent implements OnInit { this.cancelServerEdit(); this.loadServers(); }, - error: () => this.snack.open('Failed to add server', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to add server'), }); } } @@ -1542,7 +1543,7 @@ export class SettingsViewComponent implements OnInit { testServer(id: string): void { this.api.post<{ success: boolean; message: string }>(`/config/servers/${id}/test`).subscribe({ next: (r) => this.snack.open(r.message, 'Close', { duration: 3000 }), - error: () => this.snack.open('Test failed', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Test failed'), }); } @@ -1559,7 +1560,7 @@ export class SettingsViewComponent implements OnInit { .post<{ success: boolean; message: string }>(`/config/servers/test-config`, body) .subscribe({ next: (r) => this.snack.open(r.message, 'Close', { duration: 4000 }), - error: () => this.snack.open('Test failed', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Test failed'), }); } @@ -1574,7 +1575,7 @@ export class SettingsViewComponent implements OnInit { duration: 2000, }); }, - error: () => this.snack.open('Failed to update server', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to update server'), }); } @@ -1593,7 +1594,7 @@ export class SettingsViewComponent implements OnInit { this.loadServers(); this.snack.open('Server removed', 'Close', { duration: 2000 }); }, - error: () => this.snack.open('Failed to delete server', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to delete server'), }); }); } @@ -1635,7 +1636,7 @@ export class SettingsViewComponent implements OnInit { this.cancelCategoryEdit(); this.loadCategories(); }, - error: () => this.snack.open('Failed to update category', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to update category'), }); } else { this.api.post('/config/categories', cat).subscribe({ @@ -1644,7 +1645,7 @@ export class SettingsViewComponent implements OnInit { this.cancelCategoryEdit(); this.loadCategories(); }, - error: () => this.snack.open('Failed to add category', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to add category'), }); } } @@ -1665,7 +1666,7 @@ export class SettingsViewComponent implements OnInit { this.loadCategories(); this.snack.open('Category removed', 'Close', { duration: 2000 }); }, - error: () => this.snack.open('Failed to delete category', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to delete category'), }); }); } @@ -1718,7 +1719,7 @@ export class SettingsViewComponent implements OnInit { loadSabApiKey(): void { this.api.get('/config/sabnzbd-api-key').subscribe({ next: (response) => (this.sabApiKey = response.api_key), - error: () => this.snack.open('Failed to load SABnzbd API key', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to load SABnzbd API key'), }); } @@ -1741,7 +1742,7 @@ export class SettingsViewComponent implements OnInit { this.showSabApiKey = true; this.snack.open(`SABnzbd API key ${action.toLowerCase()}d`, 'Close', { duration: 2500 }); }, - error: () => this.snack.open(`Failed to ${action.toLowerCase()} SABnzbd API key`, 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, `Failed to ${action.toLowerCase()} SABnzbd API key`), }); }); } @@ -1754,14 +1755,14 @@ export class SettingsViewComponent implements OnInit { }) .subscribe({ next: () => this.snack.open('Disk guards saved', 'Close', { duration: 2000 }), - error: () => this.snack.open('Failed to save disk guards', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to save disk guards'), }); } saveSpeedLimit(): void { this.api.put('/config/speed-limit', { speed_limit_bps: this.speedLimit }).subscribe({ next: () => this.snack.open('Speed limit saved', 'Close', { duration: 2000 }), - error: () => this.snack.open('Failed to save speed limit', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to save speed limit'), }); } @@ -1770,14 +1771,14 @@ export class SettingsViewComponent implements OnInit { .put('/config/max-active-downloads', { max_active_downloads: this.maxActiveDownloads }) .subscribe({ next: () => this.snack.open('Max downloads saved', 'Close', { duration: 2000 }), - error: () => this.snack.open('Failed to save max downloads', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to save max downloads'), }); } saveRetention(): void { this.api.put('/config/history-retention', { retention: this.historyRetention }).subscribe({ next: () => this.snack.open('History retention saved', 'Close', { duration: 2000 }), - error: () => this.snack.open('Failed to save retention', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to save retention'), }); } @@ -1825,8 +1826,7 @@ export class SettingsViewComponent implements OnInit { 'Close', { duration: 2500 }, ), - error: () => - this.snack.open('Failed to save Media Library settings', 'Close', { duration: 3000 }), + error: (err) => showHttpError(this.snack, err, 'Failed to save Media Library settings'), }); } diff --git a/apps/rustnzb/frontend/src/app/features/welcome/welcome.component.ts b/apps/rustnzb/frontend/src/app/features/welcome/welcome.component.ts index 780a98da..0d7a80d2 100644 --- a/apps/rustnzb/frontend/src/app/features/welcome/welcome.component.ts +++ b/apps/rustnzb/frontend/src/app/features/welcome/welcome.component.ts @@ -3,6 +3,7 @@ import { CommonModule } from '@angular/common'; import { FormsModule } from '@angular/forms'; import { Router } from '@angular/router'; import { ApiService } from '../../core/services/api.service'; +import { httpErrorDetail } from '../../core/http/http-error'; type Step = 'landing' | 'connect' | 'preview' | 'applying'; type ImportMethod = 'api' | 'ini'; @@ -468,7 +469,7 @@ export class WelcomeComponent implements OnInit { error: (err) => { this.fetching.set(false); this.connectError.set( - err.error?.message ?? err.error?.error ?? 'Failed to connect to SABnzbd. Check the URL and API key.' + httpErrorDetail(err) ?? 'Failed to connect to SABnzbd. Check the URL and API key.' ); }, }); @@ -492,7 +493,7 @@ export class WelcomeComponent implements OnInit { error: (err) => { this.fetching.set(false); this.connectError.set( - err.error?.message ?? err.error?.error ?? 'Failed to parse the ini file.' + httpErrorDetail(err) ?? 'Failed to parse the ini file.' ); }, }); @@ -535,7 +536,7 @@ export class WelcomeComponent implements OnInit { }, error: (err) => { this.applyError.set( - err.error?.message ?? err.error?.error ?? 'Failed to apply settings.' + httpErrorDetail(err) ?? 'Failed to apply settings.' ); this.step.set('preview'); }, diff --git a/apps/rustnzb/src/group_handlers.rs b/apps/rustnzb/src/group_handlers.rs index 4edac6dc..888f3fac 100644 --- a/apps/rustnzb/src/group_handlers.rs +++ b/apps/rustnzb/src/group_handlers.rs @@ -125,7 +125,7 @@ pub async fn h_group_get( .queue_manager .with_db(|db| db.group_get(id)) .map_err(ApiError::from)? - .ok_or_else(|| ApiError::from(anyhow::anyhow!("Group not found")))?; + .ok_or_else(|| ApiError::not_found("Group not found"))?; Ok(Json(serde_json::to_value(group).map_err(|e| { ApiError::from(anyhow::anyhow!("Serialisation error: {e}")) })?)) @@ -140,7 +140,7 @@ pub async fn h_group_status( let group = qm .with_db(|db| db.group_get(id)) .map_err(ApiError::from)? - .ok_or_else(|| ApiError::from(anyhow::anyhow!("Group not found")))?; + .ok_or_else(|| ApiError::not_found("Group not found"))?; let total_headers = qm .with_db(|db| db.header_count(id, None)) @@ -215,7 +215,7 @@ pub async fn h_header_fetch( .queue_manager .with_db(|db| db.group_get(group_id)) .map_err(ApiError::from)? - .ok_or_else(|| ApiError::from(anyhow::anyhow!("Group not found")))?; + .ok_or_else(|| ApiError::not_found("Group not found"))?; let servers = state.queue_manager.get_servers(); let server = servers @@ -422,7 +422,7 @@ pub async fn h_header_download( .queue_manager .with_db(|db| db.group_get(group_id)) .map_err(ApiError::from)? - .ok_or_else(|| ApiError::from(anyhow::anyhow!("Group not found")))?; + .ok_or_else(|| ApiError::not_found("Group not found"))?; let name = input .name diff --git a/apps/rustnzb/src/handlers.rs b/apps/rustnzb/src/handlers.rs index 37f5c888..8d483a14 100644 --- a/apps/rustnzb/src/handlers.rs +++ b/apps/rustnzb/src/handlers.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use axum::Json; +use axum::extract::rejection::JsonRejection; use axum::extract::{Multipart, Path, Query, State}; use axum::http::HeaderMap; use axum::response::IntoResponse; @@ -19,6 +20,7 @@ static HTTP_CLIENT: std::sync::LazyLock = std::sync::LazyLock:: .expect("Failed to build shared HTTP client") }); +use nzb_web::nzb_core::NzbError; #[cfg(feature = "webdav")] use nzb_web::nzb_core::config::DavConfig; use nzb_web::nzb_core::config::{ @@ -27,7 +29,9 @@ use nzb_web::nzb_core::config::{ use nzb_web::nzb_core::models::*; use nzb_web::nzb_core::nzb_parser; use nzb_web::nzb_core::sabnzbd_import; +use nzb_web::queue_manager::PauseReason; +use nzb_web::HistoryRetryOutcome; use nzb_web::error::ApiError; use nzb_web::fetch_guard::{ FetchPolicy, MAX_FETCH_BODY_BYTES, build_fetch_client, check_fetch_url_allowed, @@ -96,7 +100,14 @@ pub struct MoveJobBody { #[derive(Deserialize)] pub struct SortQueueBody { - /// Sort in ascending remaining percentage order when true. + /// Field to sort by: `remaining` (default), `name`, `size`, `age` or + /// `priority`. + #[serde(default)] + pub sort_by: Option, + /// `asc` or `desc`. Takes precedence over `ascending` when present. + #[serde(default)] + pub direction: Option, + /// Legacy direction flag, used when `direction` is absent. #[serde(default = "default_sort_ascending")] pub ascending: bool, } @@ -126,10 +137,21 @@ pub struct SetPriorityBody { #[derive(Serialize)] pub struct QueueResponse { - pub jobs: Vec, + pub jobs: Vec, pub total: usize, pub speed_bps: u64, pub paused: bool, + /// Why downloads are globally paused (`global`, `disk_space`), or null. + pub pause_reason: Option, +} + +/// A queued job plus why it is paused (`manual`, `global`, `disk_space`, +/// `server`), or null when it is not paused. +#[derive(Serialize)] +pub struct QueueJob { + #[serde(flatten)] + pub job: NzbJob, + pub pause_reason: Option, } #[derive(Serialize)] @@ -206,6 +228,8 @@ pub struct AddNzbResponse { pub struct StatusResponse { pub version: &'static str, pub paused: bool, + /// Why downloads are globally paused (`global`, `disk_space`), or null. + pub pause_reason: Option, pub speed_bps: u64, pub speed_limit_bps: u64, pub queue_size: usize, @@ -272,57 +296,157 @@ pub async fn h_queue_list( let total = all_jobs.len(); let speed_bps = qm.get_speed(); let paused = qm.is_paused(); + let pause_reason = qm.global_pause_reason(); // Apply pagination (default: first 100 jobs) let offset = q.offset.unwrap_or(0); let limit = q.limit.unwrap_or(100); - let jobs: Vec<_> = all_jobs.into_iter().skip(offset).take(limit).collect(); + let jobs: Vec<_> = all_jobs + .into_iter() + .skip(offset) + .take(limit) + .map(|job| QueueJob { + pause_reason: qm.pause_reason(&job.id), + job: mask_job_password(job), + }) + .collect(); Ok(Json(QueueResponse { jobs, total, speed_bps, paused, + pause_reason, })) } -/// Enqueue a single NZB from raw bytes, applying category/priority from query params. -async fn next_uploaded_file( - multipart: &mut Multipart, -) -> Result)>, ApiError> { - let Some(field) = multipart - .next_field() - .await - .map_err(|error| ApiError::from(anyhow::anyhow!("Multipart error: {error}")))? - else { - return Ok(None); - }; - let file_name = field - .file_name() - .map(str::to_string) - .unwrap_or_else(|| "unknown.nzb".to_string()); - let data = field - .bytes() - .await - .map_err(|error| ApiError::from(anyhow::anyhow!("Read error: {error}")))?; - Ok(Some((file_name, data.to_vec()))) +/// A parsed `POST /api/queue/add` multipart body. +#[derive(Default)] +struct UploadForm { + files: Vec<(String, Vec)>, + /// The optional archive `password` text field. + password: Option, + category: Option, + priority: Option, + name: Option, +} + +impl UploadForm { + /// The query options with any form fields laid over them. + fn options(&self, q: &AddNzbQuery) -> AddNzbQuery { + AddNzbQuery { + category: self.category.clone().or_else(|| q.category.clone()), + priority: self.priority.or(q.priority), + name: self.name.clone().or_else(|| q.name.clone()), + } + } +} + +/// Read a whole `POST /api/queue/add` body. Parts that carry a file name +/// (or are named `file`) are NZB uploads; text fields `password`, `category` (alias `cat`), `priority` +/// and `name` (alias `nzbname`) set job options, in any order relative to +/// the files. Other text fields are ignored. +async fn read_upload(multipart: &mut Multipart) -> Result { + let mut form = UploadForm::default(); + while let Some(field) = multipart.next_field().await.map_err(ApiError::from)? { + let key = field.name().unwrap_or_default().to_string(); + // A part with a file name is an upload. So is a bare `file` part + // (curl's `-F "file= Some(file_name.to_string()), + None if key == "file" => Some("unknown.nzb".to_string()), + None => None, + }; + if let Some(file_name) = file_name { + let data = field.bytes().await.map_err(ApiError::from)?; + form.files.push((file_name, data.to_vec())); + continue; + } + let text = field.text().await.map_err(ApiError::from)?; + match key.as_str() { + "password" => form.password = Some(text), + "category" | "cat" => form.category = Some(text), + "name" | "nzbname" => form.name = Some(text), + "priority" => { + form.priority = Some( + text.trim() + .parse() + .map_err(|_| ApiError::bad_request("priority must be an integer"))?, + ) + } + _ => {} + } + } + Ok(form) +} + +/// Replace a job's archive password with [`PASSWORD_MASK`] (or `""` when it +/// has none) for the native API, as server passwords are masked in config +/// responses. The SABnzbd layer keeps the real value: its queue slot +/// contract includes `password`. No native endpoint accepts a job password +/// back, so the mask can never be stored in place of the real one. +fn mask_job_password(mut job: NzbJob) -> NzbJob { + job.password = Some(if job.password.is_some_and(|pw| !pw.is_empty()) { + PASSWORD_MASK.to_string() + } else { + String::new() + }); + job +} + +/// An archive that cannot be unpacked into NZBs (corrupt, empty, over the +/// size or count limits) is bad input, not a server fault. +fn bad_upload(error: anyhow::Error) -> ApiError { + ApiError::from((StatusCode::BAD_REQUEST, error)) +} + +/// Split SABnzbd's inline job password (`name{{pw}}`, `name/pw`) off a +/// name; an empty inline password counts as none. +fn take_inline_password(raw: &str) -> (String, Option) { + match nzb_web::nzb_core::path::split_job_password(raw) { + Some((name, password)) => (name, Some(password).filter(|pw| !pw.is_empty())), + None => (raw.to_string(), None), + } } +/// Enqueue a single NZB from raw bytes, applying the category, priority and +/// name override from `q`. The archive password is the explicit `password` if given, +/// else an inline one from the `name` override or the file name (SABnzbd's +/// `name{{password}}` / `name/password` convention), else the NZB's own +/// ``. fn enqueue_nzb( state: &AppState, q: &AddNzbQuery, file_name: &str, data: Vec, + password: Option<&str>, idempotency_key: Option<&IdempotencyKey>, ) -> Result { - let name = q.name.clone().unwrap_or_else(|| { + // Split inline passwords off before parse_nzb sanitizes the name. + let (name_override, name_password) = match q.name.as_deref() { + Some(name) => { + let (name, password) = take_inline_password(name); + (Some(name), password) + } + None => (None, None), + }; + let (file_name, file_password) = take_inline_password(file_name); + let name = name_override.unwrap_or_else(|| { file_name .strip_suffix(".nzb") - .unwrap_or(file_name) + .unwrap_or(&file_name) .to_string() }); let mut job = nzb_parser::parse_nzb(&name, &data).map_err(ApiError::from)?; + if let Some(password) = password + .filter(|pw| !pw.is_empty()) + .map(str::to_string) + .or(name_password) + .or(file_password) + { + job.password = Some(password); + } if let Some(ref cat) = q.category { job.category = cat.clone(); @@ -359,6 +483,9 @@ fn enqueue_nzb( /// POST /api/queue/add -- Add NZB file(s) to the queue. /// Accepts `.nzb` files directly, or `.zip`/`.gz`/`.bz2` archives containing `.nzb` files. /// Multiple files can be uploaded in a single multipart request. +/// `category`/`cat`, `priority`, `name`/`nzbname` and `password` may be sent +/// as multipart text fields or (except `password`) query parameters; a form +/// field wins over the query parameter of the same name. pub async fn h_queue_add( State(state): State>, Query(q): Query, @@ -370,9 +497,11 @@ pub async fn h_queue_add( if let Some(idempotency_key) = idempotency_key.as_ref() { // A keyed admission binds exactly one payload, so the request must carry // exactly one NZB — a multi-NZB upload has no single job to replay. + let form = read_upload(&mut multipart).await?; + let q = form.options(&q); let mut uploaded_nzbs = Vec::new(); - while let Some((file_name, data)) = next_uploaded_file(&mut multipart).await? { - uploaded_nzbs.extend(extract_nzbs(&file_name, &data).map_err(ApiError::from)?); + for (file_name, data) in &form.files { + uploaded_nzbs.extend(extract_nzbs(file_name, data).map_err(bad_upload)?); } if uploaded_nzbs.len() != 1 { return Err(ApiError::bad_request( @@ -385,13 +514,23 @@ pub async fn h_queue_add( &q, &nzb_name, nzb_data, + form.password.as_deref(), Some(idempotency_key), )?); } else { - while let Some((file_name, data)) = next_uploaded_file(&mut multipart).await? { + let form = read_upload(&mut multipart).await?; + let q = form.options(&q); + for (file_name, data) in &form.files { // Extract NZBs (handles zip/gz/bz2 archives or plain .nzb) - for (nzb_name, nzb_data) in extract_nzbs(&file_name, &data).map_err(ApiError::from)? { - nzo_ids.push(enqueue_nzb(&state, &q, &nzb_name, nzb_data, None)?); + for (nzb_name, nzb_data) in extract_nzbs(file_name, data).map_err(bad_upload)? { + nzo_ids.push(enqueue_nzb( + &state, + &q, + &nzb_name, + nzb_data, + form.password.as_deref(), + None, + )?); } } } @@ -426,14 +565,25 @@ pub async fn h_queue_set_priority( Ok(Json(SimpleResponse { status: true })) } -/// POST /api/queue/sort -- Stable sort by remaining work percentage. +/// POST /api/queue/sort -- Stable sort by `sort_by` (default: remaining +/// work percentage) in `direction` (or the legacy `ascending` flag). pub async fn h_queue_sort( State(state): State>, Json(body): Json, ) -> Result, ApiError> { - state - .queue_manager - .sort_by_remaining_percentage(body.ascending); + let field = match body.sort_by.as_deref() { + None => nzb_web::QueueSortField::Remaining, + Some(value) => value.parse().map_err(|()| { + ApiError::bad_request("sort_by must be one of: remaining, name, size, age, priority") + })?, + }; + let ascending = match body.direction.as_deref() { + None => body.ascending, + Some(value) if value.eq_ignore_ascii_case("asc") => true, + Some(value) if value.eq_ignore_ascii_case("desc") => false, + Some(_) => return Err(ApiError::bad_request("direction must be asc or desc")), + }; + state.queue_manager.sort_queue(field, ascending); Ok(Json(SimpleResponse { status: true })) } @@ -449,10 +599,26 @@ fn fetch_policy(state: &AppState) -> FetchPolicy { /// Reject an RSS feed whose URL the fetch guard would refuse, so a blocked /// feed is reported when it is saved instead of failing on every poll. +/// Reject a feed whose poll interval would hammer the indexer (and, at 0, +/// spin the RSS monitor). +fn validate_feed_poll_interval(feed: &RssFeedConfig) -> Result<(), ApiError> { + let min = nzb_web::rss_monitor::MIN_POLL_INTERVAL_SECS; + if feed.poll_interval_secs < min { + return Err(ApiError::from(( + StatusCode::BAD_REQUEST, + format!( + "poll_interval_secs must be at least {min} (got {})", + feed.poll_interval_secs + ), + ))); + } + Ok(()) +} + async fn validate_feed_url(state: &AppState, url: &str) -> Result<(), ApiError> { check_fetch_url_allowed(url, &fetch_policy(state)) .await - .map_err(|e| ApiError::from((StatusCode::BAD_REQUEST, format!("Feed URL rejected: {e}")))) + .map_err(|e| ApiError::url_rejected(format!("Feed URL rejected: {e}"))) } #[derive(Deserialize)] @@ -461,6 +627,9 @@ pub struct AddUrlBody { pub name: Option, pub category: Option, pub priority: Option, + /// Archive password; overrides inline and NZB-embedded passwords. + #[serde(default)] + pub password: Option, } /// POST /api/queue/add-url -- Add an NZB from a URL. @@ -469,7 +638,7 @@ pub async fn h_queue_add_url( Json(body): Json, ) -> Result { if body.url.is_empty() { - return Err(ApiError::from(anyhow::anyhow!("No URL provided"))); + return Err(ApiError::bad_request("No URL provided")); } let fetch_plan = validate_fetch_url_with(&body.url, &fetch_policy(&state)).await?; @@ -484,10 +653,10 @@ pub async fn h_queue_add_url( .get(fetch_plan.url.clone()) .send() .await - .map_err(|e| ApiError::from(anyhow::anyhow!("Failed to fetch URL: {e}")))?; + .map_err(|e| ApiError::bad_gateway(format!("Failed to fetch URL: {e}")))?; if !response.status().is_success() { - return Err(ApiError::from(anyhow::anyhow!( + return Err(ApiError::bad_gateway(format!( "URL returned HTTP {}", response.status() ))); @@ -511,10 +680,17 @@ pub async fn h_queue_add_url( priority: body.priority, }; - let nzbs = extract_nzbs(&file_name, &data).map_err(ApiError::from)?; + let nzbs = extract_nzbs(&file_name, &data).map_err(bad_upload)?; let mut nzo_ids = Vec::new(); for (nzb_name, nzb_data) in nzbs { - let id = enqueue_nzb(&state, &q, &nzb_name, nzb_data, None)?; + let id = enqueue_nzb( + &state, + &q, + &nzb_name, + nzb_data, + body.password.as_deref(), + None, + )?; nzo_ids.push(id); } @@ -660,54 +836,49 @@ pub async fn h_history_clear( } /// POST /api/history/{id}/retry -- Re-add a failed/completed NZB from history. +/// +/// Idempotent per history entry: while a retry of `{id}` is still queued or +/// running, a repeat request enqueues nothing and returns `200` with the +/// existing job's id in `nzo_ids` (the same replay semantics as an +/// `Idempotency-Key` queue add). Once that job has left the queue the entry +/// can be retried again. pub async fn h_history_retry( State(state): State>, Path(id): Path, ) -> Result { - // Get the history entry to get the name/category let entry = state .queue_manager .history_get(&id) .map_err(ApiError::from)? .ok_or(ApiError::not_found("History entry not found"))?; - // Get the raw NZB data - let nzb_data = state + let job_id = match state .queue_manager - .history_get_nzb_data(&id) + .retry_history_entry(&entry) .map_err(ApiError::from)? - .ok_or_else(|| ApiError::from(anyhow::anyhow!("No NZB data stored for this entry")))?; - - let qm = &state.queue_manager; - let retry_data = qm.history_get_retry_data(&id).map_err(ApiError::from)?; - let job = qm - .prepare_retry_job(&entry, &nzb_data, retry_data.as_deref()) - .map_err(ApiError::from)?; - - std::fs::create_dir_all(&job.work_dir).map_err(|e| { - ApiError::from(anyhow::anyhow!( - "Failed to create work dir '{}': {}", - job.work_dir.display(), - e - )) - })?; - - let new_id = job.id.clone(); - - tracing::info!( - name = %job.name, - id = %new_id, - original_id = %id, - "Retrying NZB from history" - ); - - qm.add_job(job, Some(nzb_data)).map_err(ApiError::from)?; + { + HistoryRetryOutcome::Started(job_id) => job_id, + HistoryRetryOutcome::InProgress(job_id) => { + tracing::info!( + id = %job_id, + original_id = %id, + "History retry already in progress" + ); + job_id + } + HistoryRetryOutcome::NoNzbData => { + return Err(ApiError::from(( + StatusCode::CONFLICT, + "No NZB data stored for this entry", + ))); + } + }; Ok(( StatusCode::OK, Json(AddNzbResponse { status: true, - nzo_ids: vec![new_id], + nzo_ids: vec![job_id], }), )) } @@ -729,6 +900,7 @@ pub async fn h_status( Ok(Json(StatusResponse { version: env!("RUSTNZB_BUILD_VERSION"), paused: qm.is_paused(), + pause_reason: qm.global_pause_reason(), speed_bps: qm.get_speed(), speed_limit_bps: qm.get_speed_limit(), queue_size: qm.queue_size(), @@ -871,6 +1043,12 @@ pub async fn h_server_add( } state.update_config_with(|config| { + if config.servers.iter().any(|s| s.id == server.id) { + return Err(ApiError::conflict(format!( + "Server '{}' already exists", + server.id + ))); + } config.servers.push(server); Ok::<_, ApiError>(()) })?; @@ -914,7 +1092,7 @@ pub async fn h_server_update( .servers .iter() .position(|s| s.id == id) - .ok_or_else(|| ApiError::from(anyhow::anyhow!("Server not found: {id}")))?; + .ok_or_else(|| ApiError::from(NzbError::ServerNotFound(id.clone())))?; let mut server = merge_server_update(&config.servers[idx], patch)?; keep_unchanged_password(&mut server, &config.servers[idx]); @@ -959,7 +1137,7 @@ pub async fn h_server_delete( config.servers.retain(|s| s.id != id); if config.servers.len() == before { - return Err(ApiError::from(anyhow::anyhow!("Server not found: {id}"))); + return Err(ApiError::from(NzbError::ServerNotFound(id.clone()))); } Ok::<_, ApiError>(()) @@ -982,7 +1160,7 @@ pub async fn h_server_test( .servers .iter() .find(|s| s.id == id) - .ok_or_else(|| ApiError::from(anyhow::anyhow!("Server not found: {id}")))? + .ok_or_else(|| ApiError::from(NzbError::ServerNotFound(id.clone())))? .clone(); // Test connection in a spawned task with timeout @@ -1133,9 +1311,10 @@ pub async fn h_category_add( State(state): State>, Json(cat): Json, ) -> Result { + cat.validate().map_err(ApiError::bad_request)?; state.update_config_with(|config| { if config.categories.iter().any(|c| c.name == cat.name) { - return Err(ApiError::from(anyhow::anyhow!( + return Err(ApiError::conflict(format!( "Category '{}' already exists", cat.name ))); @@ -1155,12 +1334,13 @@ pub async fn h_category_update( Path(name): Path, Json(cat): Json, ) -> Result, ApiError> { + cat.validate().map_err(ApiError::bad_request)?; state.update_config_with(|config| { let idx = config .categories .iter() .position(|c| c.name == name) - .ok_or_else(|| ApiError::from(anyhow::anyhow!("Category not found")))?; + .ok_or_else(|| ApiError::from(NzbError::CategoryNotFound(name.clone())))?; config.categories[idx] = cat; state .queue_manager @@ -1179,7 +1359,7 @@ pub async fn h_category_delete( let initial_len = config.categories.len(); config.categories.retain(|c| c.name != name); if config.categories.len() == initial_len { - return Err(ApiError::from(anyhow::anyhow!("Category not found"))); + return Err(ApiError::from(NzbError::CategoryNotFound(name.clone()))); } state .queue_manager @@ -1220,6 +1400,7 @@ pub async fn h_max_active_downloads_set( State(state): State>, Json(body): Json, ) -> Result, ApiError> { + validate_max_active_downloads(body.max_active_downloads)?; state.update_config_with(|config| { config.general.max_active_downloads = body.max_active_downloads; Ok::<_, ApiError>(()) @@ -1331,10 +1512,11 @@ pub async fn h_rss_feed_add( State(state): State>, Json(feed): Json, ) -> Result { + validate_feed_poll_interval(&feed)?; validate_feed_url(&state, &feed.url).await?; state.update_config_with(|config| { if config.rss_feeds.iter().any(|f| f.name == feed.name) { - return Err(ApiError::from(anyhow::anyhow!( + return Err(ApiError::conflict(format!( "Feed '{}' already exists", feed.name ))); @@ -1351,13 +1533,14 @@ pub async fn h_rss_feed_update( Path(name): Path, Json(feed): Json, ) -> Result, ApiError> { + validate_feed_poll_interval(&feed)?; validate_feed_url(&state, &feed.url).await?; state.update_config_with(|config| { let idx = config .rss_feeds .iter() .position(|f| f.name == name) - .ok_or_else(|| ApiError::from(anyhow::anyhow!("Feed not found")))?; + .ok_or_else(|| ApiError::not_found("Feed not found"))?; config.rss_feeds[idx] = feed; Ok::<_, ApiError>(()) })?; @@ -1373,7 +1556,7 @@ pub async fn h_rss_feed_delete( let len = config.rss_feeds.len(); config.rss_feeds.retain(|f| f.name != name); if config.rss_feeds.len() == len { - return Err(ApiError::from(anyhow::anyhow!("Feed not found"))); + return Err(ApiError::not_found("Feed not found")); } Ok::<_, ApiError>(()) })?; @@ -1412,12 +1595,12 @@ pub async fn h_rss_item_download( .queue_manager .rss_item_get(&id) .map_err(ApiError::from)? - .ok_or_else(|| ApiError::from(anyhow::anyhow!("RSS item not found")))?; + .ok_or_else(|| ApiError::not_found("RSS item not found"))?; let url = item .url .as_ref() - .ok_or_else(|| ApiError::from(anyhow::anyhow!("No download URL for this item")))?; + .ok_or_else(|| ApiError::from((StatusCode::CONFLICT, "No download URL for this item")))?; // Fetch the NZB let fetch_plan = validate_fetch_url_with(url, &fetch_policy(&state)).await?; @@ -1430,19 +1613,15 @@ pub async fn h_rss_item_download( .get(fetch_plan.url.clone()) .send() .await - .map_err(|e| ApiError::from(anyhow::anyhow!("Failed to fetch NZB: {e}")))?; + .map_err(|e| ApiError::bad_gateway(format!("Failed to fetch NZB: {e}")))?; if !response.status().is_success() { - return Err(ApiError::from(anyhow::anyhow!( - "HTTP {}", - response.status() - ))); + return Err(ApiError::bad_gateway(format!("HTTP {}", response.status()))); } let data = read_response_bytes_limited(response, MAX_FETCH_BODY_BYTES).await?; - let mut job = nzb_parser::parse_nzb(&item.title, &data) - .map_err(|e| ApiError::from(anyhow::anyhow!("Failed to parse NZB: {e}")))?; + let mut job = nzb_parser::parse_nzb(&item.title, &data).map_err(ApiError::from)?; // Use the item's category or feed category if let Some(ref cat) = item.category { @@ -1632,7 +1811,18 @@ pub async fn h_rss_rule_delete( // General settings handler // --------------------------------------------------------------------------- +/// Body for `PUT /api/config/general`. Every field is optional; omitted +/// fields keep their current value. +/// +/// Unknown fields are rejected with 400 rather than silently ignored, so a +/// client cannot believe it changed a setting this endpoint does not own. +/// Settings with their own endpoints stay there and are not accepted here: +/// `speed_limit_bps` (`/config/speed-limit`), `min_free_space_bytes` and +/// `abort_hopeless` (`/config/disk-guards`). `log_level` is not editable: +/// the tracing filter is built once at startup from `--log-level` / +/// `RUSTNZB_LOG_LEVEL` / `RUST_LOG`, so a saved value would have no effect. #[derive(Deserialize)] +#[serde(deny_unknown_fields)] pub struct UpdateGeneralBody { pub incomplete_dir: Option, pub complete_dir: Option, @@ -1652,13 +1842,112 @@ pub struct UpdateGeneralBody { pub script_failure: Option, pub script_timeout_secs: Option, pub script_max_output_bytes: Option, + // The settings below are read when the engine starts; changes are + // persisted immediately and take effect on the next process start. + pub required_completion_pct: Option, + pub article_timeout_secs: Option, + pub max_nested_archive_depth: Option, + pub direct_unpack: Option, + pub early_failure_check: Option, +} + +/// Article cache size bounds accepted by the API (bytes). +const CACHE_SIZE_MIN: u64 = 16 * 1024 * 1024; +const CACHE_SIZE_MAX: u64 = 64 * 1024 * 1024 * 1024; +/// Upper bound for `max_active_downloads`; `0` still means unlimited. +const MAX_ACTIVE_DOWNLOADS_LIMIT: usize = 100; +/// Upper bound for the post-processing job and worker pool sizes. +const MAX_POSTPROC_WORKERS: usize = 64; +/// `required_completion_pct` range, matching the clamp the engine applies. +const REQUIRED_COMPLETION_PCT_RANGE: std::ops::RangeInclusive = 100.0..=200.0; +/// Upper bound for `article_timeout_secs`; `0` still disables the timeout. +const ARTICLE_TIMEOUT_SECS_MAX: u64 = 3600; +/// Upper bound for `max_nested_archive_depth`. +const MAX_NESTED_ARCHIVE_DEPTH_LIMIT: u8 = 10; + +fn general_field_error(field: &str, requirement: &str) -> ApiError { + ApiError::from(( + StatusCode::BAD_REQUEST, + format!("invalid `{field}`: {requirement}"), + )) +} + +fn validate_max_active_downloads(value: usize) -> Result<(), ApiError> { + if value > MAX_ACTIVE_DOWNLOADS_LIMIT { + return Err(general_field_error( + "max_active_downloads", + &format!("must be at most {MAX_ACTIVE_DOWNLOADS_LIMIT} (0 = unlimited)"), + )); + } + Ok(()) +} + +/// Reject out-of-range values before anything is applied, so a bad request +/// never half-updates the config or the live queue manager. +fn validate_general_update(body: &UpdateGeneralBody) -> Result<(), ApiError> { + if let Some(size) = body.cache_size + && !(CACHE_SIZE_MIN..=CACHE_SIZE_MAX).contains(&size) + { + return Err(general_field_error( + "cache_size", + &format!("must be between {CACHE_SIZE_MIN} and {CACHE_SIZE_MAX} bytes"), + )); + } + if let Some(max) = body.max_active_downloads { + validate_max_active_downloads(max)?; + } + for (field, value) in [ + ("max_post_processing_jobs", body.max_post_processing_jobs), + ("max_repair_workers", body.max_repair_workers), + ("max_extract_workers", body.max_extract_workers), + ] { + // 0 is still accepted and raised to 1, as before. + if let Some(value) = value + && value > MAX_POSTPROC_WORKERS + { + return Err(general_field_error( + field, + &format!("must be at most {MAX_POSTPROC_WORKERS}"), + )); + } + } + if let Some(pct) = body.required_completion_pct + && !REQUIRED_COMPLETION_PCT_RANGE.contains(&pct) + { + return Err(general_field_error( + "required_completion_pct", + "must be between 100.0 and 200.0", + )); + } + if let Some(secs) = body.article_timeout_secs + && secs > ARTICLE_TIMEOUT_SECS_MAX + { + return Err(general_field_error( + "article_timeout_secs", + &format!("must be at most {ARTICLE_TIMEOUT_SECS_MAX} (0 = no timeout)"), + )); + } + if let Some(depth) = body.max_nested_archive_depth + && depth > MAX_NESTED_ARCHIVE_DEPTH_LIMIT + { + return Err(general_field_error( + "max_nested_archive_depth", + &format!("must be at most {MAX_NESTED_ARCHIVE_DEPTH_LIMIT}"), + )); + } + Ok(()) } /// PUT /api/config/general -- Update general settings. pub async fn h_general_update( State(state): State>, - Json(body): Json, + body: Result, JsonRejection>, ) -> Result, ApiError> { + // Map extractor failures (unknown field, wrong type) to a 400 that names + // the offending field, instead of axum's plain-text 422. + let Json(body) = + body.map_err(|rejection| ApiError::from((StatusCode::BAD_REQUEST, rejection.body_text())))?; + validate_general_update(&body)?; state.update_config_with(|config| { if let Some(dir) = body.incomplete_dir { config.general.incomplete_dir = dir.into(); @@ -1741,6 +2030,21 @@ pub async fn h_general_update( if let Some(max_output) = body.script_max_output_bytes { config.general.script_max_output_bytes = max_output; } + if let Some(pct) = body.required_completion_pct { + config.general.required_completion_pct = pct; + } + if let Some(secs) = body.article_timeout_secs { + config.general.article_timeout_secs = secs; + } + if let Some(depth) = body.max_nested_archive_depth { + config.general.max_nested_archive_depth = depth; + } + if let Some(enabled) = body.direct_unpack { + config.general.direct_unpack = enabled; + } + if let Some(enabled) = body.early_failure_check { + config.general.early_failure_check = enabled; + } state.queue_manager.set_postproc_scripts( config.general.scripts_dir.clone(), @@ -1928,7 +2232,10 @@ pub async fn h_browse_directory( let dir = std::path::Path::new(&path); if !dir.is_dir() { - return Err(ApiError::from(anyhow::anyhow!("Not a directory: {path}"))); + return Err(ApiError::from(( + StatusCode::BAD_REQUEST, + format!("Not a directory: {path}"), + ))); } let parent = dir.parent().map(|p| p.to_string_lossy().to_string()); @@ -1995,6 +2302,16 @@ pub struct BulkActionResponse { pub status: bool, pub succeeded: usize, pub failed: usize, + /// One entry per id that failed, in request order. + pub failures: Vec, +} + +#[derive(Serialize)] +pub struct BulkActionFailure { + pub id: String, + /// Same shape as an API error body (`error_kind`, `human_readable`, + /// `status`). + pub error: ApiError, } /// POST /api/queue/bulk -- Perform an action on multiple jobs. @@ -2004,12 +2321,17 @@ pub async fn h_queue_bulk_action( ) -> Result, ApiError> { let qm = &state.queue_manager; let mut succeeded = 0usize; - let mut failed = 0usize; + let mut failures = Vec::new(); for id in &body.ids { let result = match body.action.as_str() { "pause" => qm.pause_job(id), "resume" => qm.resume_job(id), + // `remove_job` is idempotent and returns Ok for an unknown id; + // a bulk request must report that id as failed. + "delete" if !qm.contains_job(id) => { + Err(nzb_web::nzb_core::NzbError::JobNotFound(id.clone())) + } "delete" => qm.remove_job(id), "priority" => { let p = body.value.as_ref().and_then(|v| v.as_i64()).unwrap_or(1) as i32; @@ -2026,14 +2348,18 @@ pub async fn h_queue_bulk_action( }; match result { Ok(_) => succeeded += 1, - Err(_) => failed += 1, + Err(error) => failures.push(BulkActionFailure { + id: id.clone(), + error: ApiError::from(error), + }), } } Ok(Json(BulkActionResponse { - status: failed == 0, + status: failures.is_empty(), succeeded, - failed, + failed: failures.len(), + failures, })) } @@ -2057,13 +2383,10 @@ pub async fn h_import_sabnzbd_ini(mut multipart: Multipart) -> Result = preview + .categories + .iter() + .filter_map(|category| { + category + .validate() + .err() + .map(|reason| format!("'{}': {reason}", category.name)) + }) + .collect(); + if !invalid.is_empty() { + return Err(ApiError::from(( + StatusCode::BAD_REQUEST, + format!("cannot apply: invalid categories: {}", invalid.join("; ")), + ))); + } + let config = state .update_config_with(|config| { // Convert imported servers → ServerConfig with fresh UUIDs diff --git a/apps/rustnzb/src/main.rs b/apps/rustnzb/src/main.rs index 2dc037ed..436896e4 100644 --- a/apps/rustnzb/src/main.rs +++ b/apps/rustnzb/src/main.rs @@ -31,9 +31,11 @@ struct Args { #[arg(long, env = "RUSTNZB_DATA_DIR")] data_dir: Option, - /// Log level (trace, debug, info, warn, error) - #[arg(long, default_value = "info", env = "RUSTNZB_LOG_LEVEL")] - log_level: String, + /// Log level (trace, debug, info, warn, error). No clap default: when + /// neither this flag nor RUSTNZB_LOG_LEVEL is given, config.toml's + /// `general.log_level` applies (falling back to "info"). + #[arg(long, env = "RUSTNZB_LOG_LEVEL")] + log_level: Option, /// Log file path #[arg(long, env = "RUSTNZB_LOG_FILE")] @@ -105,6 +107,64 @@ fn init_otel_metrics( Some(provider) } +const LOG_LEVEL_NAMES: [&str; 6] = ["trace", "debug", "info", "warn", "error", "off"]; + +/// Whether `spec` is a usable log filter: either a bare level name or a full +/// `EnvFilter` directive list (e.g. `info,nzb_nntp=debug`). A bare word that +/// is not a level is rejected, because `EnvFilter` would otherwise read it as +/// a target name and silently enable every level for that target only. +fn is_valid_log_filter(spec: &str) -> bool { + if EnvFilter::try_new(spec).is_err() { + return false; + } + spec.split(',').map(str::trim).all(|d| { + d.is_empty() + || d.contains('=') + || d.contains('[') + || LOG_LEVEL_NAMES.iter().any(|l| l.eq_ignore_ascii_case(d)) + }) +} + +/// Resolve the tracing filter. Precedence, highest first: +/// +/// 1. `RUST_LOG` (full filter override) +/// 2. `--log-level`, then `RUSTNZB_LOG_LEVEL` (clap lets the flag win) +/// 3. config.toml `general.log_level` +/// 4. `"info"` +/// +/// Empty values count as unset. An invalid value falls back to `"info"`, and +/// an invalid `RUST_LOG` falls through to the next source; either way a +/// warning is returned for the caller to log once tracing is up. +fn resolve_log_filter( + rust_log: Option<&str>, + cli_or_env: Option<&str>, + toml_level: &str, +) -> (String, Option) { + let mut warning = None; + if let Some(rust_log) = rust_log.map(str::trim).filter(|s| !s.is_empty()) { + if EnvFilter::try_new(rust_log).is_ok() { + return (rust_log.to_string(), None); + } + warning = Some(format!("ignoring invalid RUST_LOG filter {rust_log:?}")); + } + let chosen = [cli_or_env, Some(toml_level)] + .into_iter() + .flatten() + .map(str::trim) + .find(|s| !s.is_empty()); + match chosen { + Some(spec) if is_valid_log_filter(spec) => (spec.to_string(), warning), + Some(spec) => ( + "info".to_string(), + Some(format!( + "invalid log level {spec:?} (expected one of {}); falling back to \"info\"", + LOG_LEVEL_NAMES.join(", ") + )), + ), + None => ("info".to_string(), warning), + } +} + /// Verify that all external tools work in the current environment. /// Returns 0 on success, 1 on failure. fn run_smoke_tests() -> i32 { @@ -214,8 +274,13 @@ async fn main() -> anyhow::Result<()> { // Initialize logging (must happen before startup::initialize) let log_buffer = LogBuffer::new(); - let filter = - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); + let rust_log = std::env::var(EnvFilter::DEFAULT_ENV).ok(); + let (filter_spec, log_filter_warning) = resolve_log_filter( + rust_log.as_deref(), + args.log_level.as_deref(), + &config.general.log_level, + ); + let filter = EnvFilter::new(&filter_spec); let fmt_layer = tracing_subscriber::fmt::layer().with_target(true); let log_layer = LogBufferLayer::new(log_buffer.clone()); @@ -278,6 +343,9 @@ async fn main() -> anyhow::Result<()> { } info!("rustnzb v{}", env!("RUSTNZB_BUILD_VERSION")); + if let Some(warning) = log_filter_warning { + tracing::warn!("{warning}"); + } // Initialize the engine (config, DB, queue manager, background services) let result = nzb_web::startup::initialize( @@ -286,7 +354,7 @@ async fn main() -> anyhow::Result<()> { listen_addr: args.listen_addr, port: args.port, data_dir: args.data_dir, - log_level: Some(args.log_level), + log_level: Some(filter_spec), }, Some(log_buffer), ) @@ -314,7 +382,6 @@ async fn main() -> anyhow::Result<()> { #[cfg(feature = "webdav")] { - use axum::Extension; use std::sync::Arc; let servers = result.queue_manager.get_servers(); @@ -378,36 +445,7 @@ async fn main() -> anyhow::Result<()> { }); } - let router = { - let mut r = rustnzb::server::build_router(result.state.clone()); - if let Some(ref dav) = dav_handle { - let auth_state = result.state.clone(); - let dav_auth = axum::middleware::from_fn( - move |headers: axum::http::HeaderMap, - req: axum::extract::Request, - next: axum::middleware::Next| { - let st = auth_state.clone(); - async move { rustnzb::dav::auth::dav_auth(st, headers, req, next).await } - }, - ); - let dav_router = nzbdav_dav::dav_router(Arc::clone(&dav.store)).layer(dav_auth); - r = r.nest("/dav", dav_router); - let cfg = result.state.config(); - if cfg.dav.username.is_none() - && cfg.dav.password.is_none() - && cfg.dav.api_key.is_none() - { - tracing::warn!( - "WebDAV media library mounted at /dav (UNAUTHENTICATED — \ - set dav.username/password or dav.api_key in config)" - ); - } else { - info!("WebDAV media library mounted at /dav (auth enabled)"); - } - } - // Always layer Option> so h_status and h_dav_add can extract it. - r.layer(Extension(dav_handle)) - }; + let router = rustnzb::server::build_router_with_dav(result.state.clone(), dav_handle); rustnzb::server::serve(result.state, router).await?; } @@ -422,7 +460,7 @@ async fn main() -> anyhow::Result<()> { #[cfg(test)] mod tests { - use super::Args; + use super::{Args, resolve_log_filter}; use clap::Parser; use serial_test::serial; @@ -469,4 +507,94 @@ mod tests { assert_eq!(args.port, None); } + + // BUG-81: general.log_level in config.toml must take effect. Precedence is + // RUST_LOG > --log-level > RUSTNZB_LOG_LEVEL > TOML general.log_level > "info". + + #[test] + fn rust_log_wins_over_everything() { + let (f, w) = resolve_log_filter(Some("nzb_nntp=trace"), Some("warn"), "debug"); + assert_eq!(f, "nzb_nntp=trace"); + assert!(w.is_none()); + } + + #[test] + fn explicit_level_wins_over_toml() { + let (f, w) = resolve_log_filter(None, Some("warn"), "debug"); + assert_eq!(f, "warn"); + assert!(w.is_none()); + } + + #[test] + fn toml_level_used_when_no_cli_or_env() { + let (f, w) = resolve_log_filter(None, None, "debug"); + assert_eq!(f, "debug"); + assert!(w.is_none()); + } + + #[test] + fn defaults_to_info_when_nothing_set() { + let (f, w) = resolve_log_filter(None, None, ""); + assert_eq!(f, "info"); + assert!(w.is_none()); + let (f, _) = resolve_log_filter(Some(" "), Some(""), " "); + assert_eq!(f, "info"); + } + + #[test] + fn full_filter_directives_are_accepted() { + let (f, w) = resolve_log_filter(None, None, "info,nzb_nntp=debug"); + assert_eq!(f, "info,nzb_nntp=debug"); + assert!(w.is_none()); + } + + #[test] + fn invalid_level_falls_back_to_info_with_warning() { + let (f, w) = resolve_log_filter(None, None, "verbose"); + assert_eq!(f, "info"); + assert!(w.unwrap().contains("verbose")); + + let (f, w) = resolve_log_filter(None, Some("loud"), "debug"); + assert_eq!(f, "info"); + assert!(w.unwrap().contains("loud")); + } + + #[test] + fn invalid_rust_log_falls_through_to_next_source() { + let (f, w) = resolve_log_filter(Some("=[bad"), None, "debug"); + assert_eq!(f, "debug"); + assert!(w.unwrap().contains("RUST_LOG")); + } + + #[test] + fn level_names_are_case_insensitive() { + let (f, w) = resolve_log_filter(None, None, "DEBUG"); + assert_eq!(f, "DEBUG"); + assert!(w.is_none()); + } + + #[test] + #[serial(rustnzb_log_level_env)] + fn log_level_arg_has_no_clap_default() { + unsafe { + std::env::remove_var("RUSTNZB_LOG_LEVEL"); + } + let args = Args::parse_from(["rustnzb"]); + assert_eq!(args.log_level, None); + } + + #[test] + #[serial(rustnzb_log_level_env)] + fn log_level_env_var_is_used_and_explicit_flag_wins() { + unsafe { + std::env::set_var("RUSTNZB_LOG_LEVEL", "debug"); + } + let from_env = Args::parse_from(["rustnzb"]); + let from_flag = Args::parse_from(["rustnzb", "--log-level", "warn"]); + unsafe { + std::env::remove_var("RUSTNZB_LOG_LEVEL"); + } + assert_eq!(from_env.log_level.as_deref(), Some("debug")); + assert_eq!(from_flag.log_level.as_deref(), Some("warn")); + } } diff --git a/apps/rustnzb/src/server.rs b/apps/rustnzb/src/server.rs index 45e5ab88..48c71689 100644 --- a/apps/rustnzb/src/server.rs +++ b/apps/rustnzb/src/server.rs @@ -115,7 +115,58 @@ fn is_hashed_asset(path: &str) -> bool { } /// Build the axum Router with all API routes. +/// +/// With the `webdav` feature this is [`build_router_with_dav`] with no DAV +/// handle, so the router always carries the `Option>` extension +/// that `/api/status` and `/api/dav/add` extract. Every caller — the binary and +/// the integration tests — gets the same wiring. pub fn build_router(state: Arc) -> Router { + #[cfg(feature = "webdav")] + { + build_router_with_dav(state, None) + } + #[cfg(not(feature = "webdav"))] + { + build_api_router(state) + } +} + +/// Build the router with an optional WebDAV media library. +/// +/// When `dav` is `Some`, the DAV store is mounted at `/dav` behind +/// [`crate::dav::auth::dav_auth`]. The `Option>` extension is +/// layered either way, because handlers extract it unconditionally when the +/// feature is compiled in and would otherwise fail with a 500. +#[cfg(feature = "webdav")] +pub fn build_router_with_dav( + state: Arc, + dav: Option>, +) -> Router { + let mut router = build_api_router(state.clone()); + if let Some(ref dav) = dav { + let auth_state = state.clone(); + let dav_auth = axum::middleware::from_fn( + move |headers: HeaderMap, req: axum::extract::Request, next: Next| { + let st = auth_state.clone(); + async move { crate::dav::auth::dav_auth(st, headers, req, next).await } + }, + ); + let dav_router = nzbdav_dav::dav_router(Arc::clone(&dav.store)).layer(dav_auth); + router = router.nest("/dav", dav_router); + let cfg = state.config(); + if cfg.dav.username.is_none() && cfg.dav.password.is_none() && cfg.dav.api_key.is_none() { + tracing::warn!( + "WebDAV media library mounted at /dav (UNAUTHENTICATED — \ + set dav.username/password or dav.api_key in config)" + ); + } else { + info!("WebDAV media library mounted at /dav (auth enabled)"); + } + } + router.layer(axum::Extension(dav)) +} + +fn build_api_router(state: Arc) -> Router { let cors = CorsLayer::default() .allow_origin(AllowOrigin::any()) .allow_headers(AllowHeaders::any()); diff --git a/apps/rustnzb/tests/api_contracts.rs b/apps/rustnzb/tests/api_contracts.rs index c576691b..d4c8a2e1 100644 --- a/apps/rustnzb/tests/api_contracts.rs +++ b/apps/rustnzb/tests/api_contracts.rs @@ -283,7 +283,9 @@ async fn config_routes_validate_duplicates_and_persist_successful_updates() { .await .unwrap() .status(), - reqwest::StatusCode::INTERNAL_SERVER_ERROR + // A duplicate name is a client-side conflict. This asserted 500 + // before BUG-73 routed it through the central 409 mapping. + reqwest::StatusCode::CONFLICT ); let feed = serde_json::json!({"name":"daily", "url":"https://example.test/feed", "poll_interval_secs":60, "category":"tv", "filter_regex":null, "enabled":true, "auto_download":false}); @@ -461,6 +463,193 @@ async fn rss_feed_url_is_validated_when_saved() { ); } +#[tokio::test] +async fn rss_feed_poll_interval_below_floor_is_rejected_when_saved() { + let app = start_app(false).await; + let client = reqwest::Client::new(); + let access = setup_access(&app, &client).await; + let feed = |secs: u64| serde_json::json!({"name":"tight", "url":"https://example.test/feed", "poll_interval_secs": secs, "enabled": true}); + + for secs in [0, 1, 59] { + let rejected = client + .post(format!("{}/api/config/rss-feeds", app.base_url)) + .bearer_auth(&access) + .json(&feed(secs)) + .send() + .await + .unwrap(); + assert_eq!( + rejected.status(), + reqwest::StatusCode::BAD_REQUEST, + "{secs}s" + ); + assert!( + rejected + .text() + .await + .unwrap() + .contains("poll_interval_secs") + ); + } + assert!(app.state.config().rss_feeds.is_empty()); + + assert_eq!( + client + .post(format!("{}/api/config/rss-feeds", app.base_url)) + .bearer_auth(&access) + .json(&feed(60)) + .send() + .await + .unwrap() + .status(), + reqwest::StatusCode::OK + ); + let rejected = client + .put(format!("{}/api/config/rss-feeds/tight", app.base_url)) + .bearer_auth(&access) + .json(&feed(0)) + .send() + .await + .unwrap(); + assert_eq!(rejected.status(), reqwest::StatusCode::BAD_REQUEST); + assert_eq!(app.state.config().rss_feeds[0].poll_interval_secs, 60); +} + +/// Serve one HTTP response with `status` and an empty body on a loopback +/// port and return its base URL. +async fn serve_status_once(status: &'static str) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let mut request = [0; 2048]; + let _ = socket.read(&mut request).await; + let head = + format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.shutdown().await; + } + }); + format!("http://{address}") +} + +/// A URL the fetch guard refuses is client input: every native endpoint that +/// fetches a caller-supplied URL answers 400 `url_rejected`, never 500. +#[tokio::test] +async fn fetch_guard_rejections_are_400_url_rejected() { + let app = start_app(false).await; + let client = reqwest::Client::new(); + let access = setup_access(&app, &client).await; + + let rejected_urls = [ + "http://127.0.0.1:9/", + "http://192.168.1.10/", + "http://10.0.0.1/", + "http://169.254.169.254/latest", + "file:///etc/passwd", + "not a url", + ]; + + for url in rejected_urls { + let (status, body) = call( + client + .post(format!("{}/api/setup/import-sabnzbd-api", app.base_url)) + .bearer_auth(&access) + .json(&serde_json::json!({"url": url, "api_key": "k"})), + ) + .await; + assert_eq!(status, 400, "import-sabnzbd-api {url}: {body}"); + assert_eq!(body["error_kind"], "url_rejected", "{url}: {body}"); + + let (status, body) = call( + client + .post(format!("{}/api/queue/add-url", app.base_url)) + .bearer_auth(&access) + .json(&serde_json::json!({ "url": url })), + ) + .await; + assert_eq!(status, 400, "add-url {url}: {body}"); + assert_eq!(body["error_kind"], "url_rejected", "{url}: {body}"); + + let (status, body) = call( + client + .post(format!("{}/api/config/rss-feeds", app.base_url)) + .bearer_auth(&access) + .json(&serde_json::json!({"name": "f", "url": url, "enabled": true})), + ) + .await; + assert_eq!(status, 400, "rss-feeds {url}: {body}"); + assert_eq!(body["error_kind"], "url_rejected", "{url}: {body}"); + } + + // An empty add-url body is also client input. + let (status, _) = call( + client + .post(format!("{}/api/queue/add-url", app.base_url)) + .bearer_auth(&access) + .json(&serde_json::json!({ "url": "" })), + ) + .await; + assert_eq!(status, 400); + + // RSS item download re-validates the stored item URL. + app.state + .queue_manager + .rss_item_upsert(&nzb_web::nzb_core::models::RssItem { + id: "item-1".into(), + feed_name: "f".into(), + title: "Item".into(), + url: Some("http://169.254.169.254/latest".into()), + published_at: None, + first_seen_at: chrono::Utc::now(), + downloaded: false, + downloaded_at: None, + category: None, + size_bytes: 0, + }) + .unwrap(); + let (status, body) = call( + client + .post(format!("{}/api/rss/items/item-1/download", app.base_url)) + .bearer_auth(&access), + ) + .await; + assert_eq!(status, 400, "rss item download: {body}"); + assert_eq!(body["error_kind"], "url_rejected", "{body}"); +} + +/// When the guard admits a URL but the remote answers with an error status, +/// the failure is the upstream's: 502 Bad Gateway rather than 500. +#[tokio::test] +async fn upstream_error_statuses_are_502() { + let app = start_app(false).await; + let client = reqwest::Client::new(); + let access = setup_access(&app, &client).await; + set_fetch_policy(&app, false, &["127.0.0.0/8"]); + + let base = serve_status_once("500 Internal Server Error").await; + let (status, body) = call( + client + .post(format!("{}/api/queue/add-url", app.base_url)) + .bearer_auth(&access) + .json(&serde_json::json!({ "url": format!("{base}/release.nzb") })), + ) + .await; + assert_eq!(status, 502, "add-url: {body}"); + assert_eq!(body["error_kind"], "bad_gateway", "{body}"); + + let (status, body) = call( + client + .post(format!("{}/api/setup/import-sabnzbd-api", app.base_url)) + .bearer_auth(&access) + .json(&serde_json::json!({"url": base, "api_key": "k"})), + ) + .await; + assert_eq!(status, 502, "import-sabnzbd-api: {body}"); + assert_eq!(body["error_kind"], "bad_gateway", "{body}"); +} + async fn login(app: &ContractApp, client: &reqwest::Client) -> (String, String) { let tokens = client .post(format!("{}/api/auth/login", app.base_url)) @@ -650,6 +839,17 @@ async fn unknown_ids_return_404_and_invalid_input_returns_400() { body["warnings"][0].as_str().unwrap().contains("nope"), "{body}" ); + + // An empty feed list means "all feeds", so it is saved without a warning. + let (status, body) = call( + client + .post(format!("{base}/api/rss/rules")) + .bearer_auth(&access) + .json(&serde_json::json!({"name": "all", "feed_names": [], "match_regex": ".*"})), + ) + .await; + assert_eq!(status, 200, "{body}"); + assert!(body.get("warnings").is_none(), "{body}"); } #[tokio::test] @@ -884,3 +1084,448 @@ async fn missing_article_returns_404() { .await; assert_eq!(status, 404, "{body}"); } + +/// BUG-73: config endpoints answer an unknown name or id with 404 and a +/// duplicate add with 409, instead of a 500 that reads as a server fault. +#[tokio::test] +async fn unknown_config_entries_return_404_and_duplicates_return_409() { + let app = start_app(true).await; + let client = reqwest::Client::new(); + let (access, _) = login(&app, &client).await; + let base = &app.base_url; + + // Categories. + let category = serde_json::json!({"name": "tv", "output_dir": null, "post_processing": 3}); + let (status, body) = call( + client + .put(format!("{base}/api/config/categories/no-such-cat")) + .bearer_auth(&access) + .json(&category), + ) + .await; + assert_eq!(status, 404, "{body}"); + assert_eq!(body["error_kind"], "not_found"); + let (status, body) = call( + client + .delete(format!("{base}/api/config/categories/no-such-cat")) + .bearer_auth(&access), + ) + .await; + assert_eq!(status, 404, "{body}"); + assert_eq!(body["error_kind"], "not_found"); + for expected in [200, 409] { + let (status, body) = call( + client + .post(format!("{base}/api/config/categories")) + .bearer_auth(&access) + .json(&category), + ) + .await; + assert_eq!(status, expected, "{body}"); + } + + // Servers. + let (status, body) = call( + client + .put(format!("{base}/api/config/servers/no-such-server")) + .bearer_auth(&access) + .json(&serde_json::json!({"port": 563})), + ) + .await; + assert_eq!(status, 404, "{body}"); + assert_eq!(body["error_kind"], "server_not_found"); + let (status, body) = call( + client + .delete(format!("{base}/api/config/servers/no-such-server")) + .bearer_auth(&access), + ) + .await; + assert_eq!(status, 404, "{body}"); + assert_eq!(body["error_kind"], "server_not_found"); + let (status, body) = call( + client + .post(format!("{base}/api/config/servers/no-such-server/test")) + .bearer_auth(&access), + ) + .await; + assert_eq!(status, 404, "{body}"); + let server = serde_json::json!({ + "id": "primary", "name": "Primary", "host": "news.example.test", + "port": 563, "ssl": true, "connections": 4, + }); + for expected in [200, 409] { + let (status, body) = call( + client + .post(format!("{base}/api/config/servers")) + .bearer_auth(&access) + .json(&server), + ) + .await; + assert_eq!(status, expected, "{body}"); + } + assert_eq!(app.state.config().servers.len(), 1); + + // RSS feeds and items. + let feed = serde_json::json!({"name":"daily", "url":"https://example.test/feed", "poll_interval_secs":60, "category":null, "filter_regex":null, "enabled":true, "auto_download":false}); + let (status, body) = call( + client + .put(format!("{base}/api/config/rss-feeds/no-such-feed")) + .bearer_auth(&access) + .json(&feed), + ) + .await; + assert_eq!(status, 404, "{body}"); + let (status, body) = call( + client + .delete(format!("{base}/api/config/rss-feeds/no-such-feed")) + .bearer_auth(&access), + ) + .await; + assert_eq!(status, 404, "{body}"); + for expected in [200, 409] { + let (status, body) = call( + client + .post(format!("{base}/api/config/rss-feeds")) + .bearer_auth(&access) + .json(&feed), + ) + .await; + assert_eq!(status, expected, "{body}"); + } + let (status, body) = call( + client + .post(format!("{base}/api/rss/items/no-such-item/download")) + .bearer_auth(&access), + ) + .await; + assert_eq!(status, 404, "{body}"); + + // Groups. + for path in ["", "/status"] { + let (status, body) = call( + client + .get(format!("{base}/api/groups/987654{path}")) + .bearer_auth(&access), + ) + .await; + assert_eq!(status, 404, "{path}: {body}"); + assert_eq!(body["error_kind"], "not_found"); + } + let (status, body) = call( + client + .post(format!("{base}/api/groups/987654/headers/fetch")) + .bearer_auth(&access), + ) + .await; + assert_eq!(status, 404, "{body}"); +} + +/// BUG-74: a category whose name or output_dir would make every later +/// enqueue fail ("category output path is unsafe") is refused at save time. +#[tokio::test] +async fn category_save_rejects_unsafe_name_and_output_dir() { + let app = start_app(true).await; + let client = reqwest::Client::new(); + let (access, _) = login(&app, &client).await; + let base = &app.base_url; + let category = |name: &str, output_dir: serde_json::Value| serde_json::json!({"name": name, "output_dir": output_dir, "post_processing": 3}); + + let unsafe_bodies = [ + category("../escape", serde_json::Value::Null), + category("a/b", serde_json::Value::Null), + category("..", serde_json::Value::Null), + category("", serde_json::Value::Null), + category("tab\tname", serde_json::Value::Null), + category("tv", serde_json::json!("../../../etc")), + category("tv", serde_json::json!("shows/../../etc")), + category("tv", serde_json::json!("")), + category("tv", serde_json::json!("/srv/media/../../etc")), + ]; + for body in &unsafe_bodies { + let (status, response) = call( + client + .post(format!("{base}/api/config/categories")) + .bearer_auth(&access) + .json(body), + ) + .await; + assert_eq!(status, 400, "POST {body}: {response}"); + assert_eq!(response["error_kind"], "bad_request"); + let (status, response) = call( + client + .put(format!("{base}/api/config/categories/Default")) + .bearer_auth(&access) + .json(body), + ) + .await; + assert_eq!(status, 400, "PUT {body}: {response}"); + } + let saved = AppConfig::load(&app.config_path).unwrap(); + assert_eq!(saved.categories.len(), 1); + assert_eq!(saved.categories[0].name, "Default"); + assert!(saved.categories[0].output_dir.is_none()); + + // What enqueue accepts is still accepted: relative subdirectories and + // absolute roots. + for body in [ + category("tv", serde_json::json!("shows/tv")), + category("movies", serde_json::json!("/srv/media/movies")), + ] { + let (status, response) = call( + client + .post(format!("{base}/api/config/categories")) + .bearer_auth(&access) + .json(&body), + ) + .await; + assert_eq!(status, 200, "{body}: {response}"); + } + let output = app + .state + .queue_manager + .output_dir_for("tv", "Some.Job") + .unwrap(); + assert!(output.ends_with("complete/shows/tv/Some.Job"), "{output:?}"); +} + +#[tokio::test] +async fn bad_input_on_add_browse_and_rss_download_is_a_client_error() { + let app = start_app(true).await; + let client = reqwest::Client::new(); + let (access, _) = login(&app, &client).await; + let base = &app.base_url; + + // Empty URL on add-url is the caller's mistake, not a server fault. + let (status, body) = call( + client + .post(format!("{base}/api/queue/add-url")) + .bearer_auth(&access) + .json(&serde_json::json!({"url": ""})), + ) + .await; + assert_eq!(status, 400, "{body}"); + assert_eq!(body["error_kind"], "bad_request"); + + // Browsing a path that is missing, or is a file, is a 400. + let missing = app.config_path.with_file_name("no-such-dir"); + let (status, body) = call( + client + .get(format!("{base}/api/browse-directory")) + .query(&[("path", missing.to_string_lossy().as_ref())]) + .bearer_auth(&access), + ) + .await; + assert_eq!(status, 400, "{body}"); + assert_eq!(body["error_kind"], "bad_request"); + assert!( + body["human_readable"] + .as_str() + .unwrap() + .contains("Not a directory") + ); + let (status, body) = call( + client + .get(format!("{base}/api/browse-directory")) + .query(&[("path", app.config_path.to_string_lossy().as_ref())]) + .bearer_auth(&access), + ) + .await; + assert_eq!(status, 400, "{body}"); + + // A corrupt archive upload is a 400. + let part = reqwest::multipart::Part::bytes(b"not a zip".to_vec()).file_name("broken.zip"); + let (status, body) = call( + client + .post(format!("{base}/api/queue/add")) + .bearer_auth(&access) + .multipart(reqwest::multipart::Form::new().part("file", part)), + ) + .await; + assert_eq!(status, 400, "{body}"); + assert_eq!(body["error_kind"], "bad_request"); + + // A malformed multipart body is a 400. + let (status, body) = call( + client + .post(format!("{base}/api/queue/add")) + .bearer_auth(&access) + .header("content-type", "multipart/form-data; boundary=XYZ") + .body("--XYZ\r\nContent-Disposition: form-data; name=\"file\"\r\n\r\ntruncated"), + ) + .await; + assert_eq!(status, 400, "{body}"); + assert_eq!(body["error_kind"], "bad_request"); + + // Downloading an RSS item that has no NZB URL conflicts with its state. + app.state + .queue_manager + .rss_item_upsert(&nzb_web::nzb_core::models::RssItem { + id: "no-url".into(), + feed_name: "feed".into(), + title: "Item".into(), + url: None, + published_at: None, + first_seen_at: chrono::Utc::now(), + downloaded: false, + downloaded_at: None, + category: None, + size_bytes: 0, + }) + .unwrap(); + let (status, body) = call( + client + .post(format!("{base}/api/rss/items/no-url/download")) + .bearer_auth(&access), + ) + .await; + assert_eq!(status, 409, "{body}"); + assert_eq!(body["error_kind"], "conflict"); +} + +/// BUG-80: PUT /api/config/general used to answer `{"status":true}` for +/// fields it silently dropped and for nonsensical values. +#[tokio::test] +async fn general_config_rejects_unknown_fields_and_out_of_range_values() { + let app = start_app(true).await; + let client = reqwest::Client::new(); + let (access, _) = login(&app, &client).await; + let url = format!("{}/api/config/general", app.base_url); + let before = AppConfig::load(&app.config_path).unwrap(); + + let rejected = [ + // Unknown / non-editable fields name the offending key. + (serde_json::json!({"no_such_setting": 1}), "no_such_setting"), + (serde_json::json!({"log_level": "debug"}), "log_level"), + ( + serde_json::json!({"abort_hopeless": false}), + "abort_hopeless", + ), + // Out-of-range values. + (serde_json::json!({"cache_size": 0}), "cache_size"), + (serde_json::json!({"cache_size": 1_u64 << 40}), "cache_size"), + ( + serde_json::json!({"max_active_downloads": 99_999}), + "max_active_downloads", + ), + ( + serde_json::json!({"max_post_processing_jobs": 999}), + "max_post_processing_jobs", + ), + ( + serde_json::json!({"max_repair_workers": 999}), + "max_repair_workers", + ), + ( + serde_json::json!({"max_extract_workers": 999}), + "max_extract_workers", + ), + ( + serde_json::json!({"required_completion_pct": 99.0}), + "required_completion_pct", + ), + ( + serde_json::json!({"required_completion_pct": 250.0}), + "required_completion_pct", + ), + ( + serde_json::json!({"article_timeout_secs": 86_400}), + "article_timeout_secs", + ), + ( + serde_json::json!({"max_nested_archive_depth": 11}), + "max_nested_archive_depth", + ), + (serde_json::json!({"direct_unpack": "yes"}), "direct_unpack"), + ]; + for (payload, field) in rejected { + let (status, body) = call(client.put(&url).bearer_auth(&access).json(&payload)).await; + assert_eq!(status, 400, "{payload} -> {body}"); + assert_eq!(body["error_kind"], "bad_request", "{payload} -> {body}"); + assert!( + body["human_readable"] + .as_str() + .unwrap_or("") + .contains(field), + "{payload}: error should name `{field}`: {body}" + ); + } + + // A rejected request must not apply any part of itself: the valid + // `direct_unpack` change is discarded along with the bad cache size. + let (status, body) = call( + client + .put(&url) + .bearer_auth(&access) + .json(&serde_json::json!({"direct_unpack": false, "cache_size": 0})), + ) + .await; + assert_eq!(status, 400, "{body}"); + + let after = AppConfig::load(&app.config_path).unwrap(); + assert_eq!( + serde_json::to_value(&after.general).unwrap(), + serde_json::to_value(&before.general).unwrap(), + "rejected updates must not change the saved config" + ); + assert_eq!(app.state.queue_manager.get_max_active_downloads(), 1); +} + +/// BUG-80: settings that previously had no API path can now be edited, and +/// the documented sentinel values (0 = unlimited / no timeout / outer +/// archive only) are accepted. +#[tokio::test] +async fn general_config_persists_previously_unreachable_settings() { + let app = start_app(true).await; + let client = reqwest::Client::new(); + let (access, _) = login(&app, &client).await; + let url = format!("{}/api/config/general", app.base_url); + + let (status, body) = call( + client + .put(&url) + .bearer_auth(&access) + .json(&serde_json::json!({ + "required_completion_pct": 150.0, + "article_timeout_secs": 0, + "max_nested_archive_depth": 0, + "direct_unpack": false, + "early_failure_check": false, + "cache_size": 16 * 1024 * 1024, + "max_active_downloads": 0, + "max_post_processing_jobs": 64, + "max_repair_workers": 2, + "max_extract_workers": 3, + })), + ) + .await; + assert_eq!(status, 200, "{body}"); + assert_eq!(body["status"], true); + + for general in [ + AppConfig::load(&app.config_path).unwrap().general, + app.state.config().general.clone(), + ] { + assert_eq!(general.required_completion_pct, 150.0); + assert_eq!(general.article_timeout_secs, 0); + assert_eq!(general.max_nested_archive_depth, 0); + assert!(!general.direct_unpack); + assert!(!general.early_failure_check); + assert_eq!(general.cache_size, 16 * 1024 * 1024); + assert_eq!(general.max_active_downloads, 0); + assert_eq!(general.max_post_processing_jobs, 64); + assert_eq!(general.max_repair_workers, 2); + assert_eq!(general.max_extract_workers, 3); + } + assert_eq!(app.state.queue_manager.get_max_active_downloads(), 0); + + // The dedicated endpoint shares the same bound. + let (status, body) = call( + client + .put(format!("{}/api/config/max-active-downloads", app.base_url)) + .bearer_auth(&access) + .json(&serde_json::json!({"max_active_downloads": 99_999})), + ) + .await; + assert_eq!(status, 400, "{body}"); + assert_eq!(app.state.queue_manager.get_max_active_downloads(), 0); +} diff --git a/apps/rustnzb/tests/disk_space_pause_api.rs b/apps/rustnzb/tests/disk_space_pause_api.rs new file mode 100644 index 00000000..9aa87d94 --- /dev/null +++ b/apps/rustnzb/tests/disk_space_pause_api.rs @@ -0,0 +1,174 @@ +//! The disk-space guard's pause must be distinguishable from a user pause +//! in the native API and the SABnzbd compatibility layer. + +use std::sync::Arc; + +use arc_swap::ArcSwap; +use axum::body::to_bytes; +use axum::extract::{Query, State}; +use axum::response::IntoResponse; +use nzb_web::auth::{CredentialStore, TokenStore}; +use nzb_web::log_buffer::LogBuffer; +use nzb_web::nzb_core::config::AppConfig; +use nzb_web::nzb_core::db::Database; +use nzb_web::nzb_core::nzb_parser; +use nzb_web::queue_manager::QueueManager; +use nzb_web::sabnzbd_compat::{SabApiRequest, h_sabnzbd_api_get}; +use nzb_web::state::AppState; +use rustnzb::handlers::{QueueQuery, h_queue_list, h_status}; + +const NZB: &str = r#" + + + alt.binaries.test + a1@rustnzb.test + +"#; + +fn test_state() -> (Arc, tempfile::TempDir) { + let tempdir = tempfile::tempdir().expect("tempdir"); + let log_buffer = LogBuffer::default(); + let manager = QueueManager::new( + Vec::new(), + Database::open_memory().expect("database"), + tempdir.path().join("incomplete"), + tempdir.path().join("complete"), + log_buffer.clone(), + 1, + Vec::new(), + 0, + 0, + false, + 5, + false, + false, + 100.0, + 30, + ); + let state = AppState::new( + Arc::new(ArcSwap::from_pointee(AppConfig::default())), + tempdir.path().join("config.toml"), + manager, + log_buffer, + Arc::new(TokenStore::new()), + Arc::new(CredentialStore::new(tempdir.path().to_path_buf())), + ); + (Arc::new(state), tempdir) +} + +/// Trip the disk guard with one job in the queue; returns the job id. +fn hold_for_disk_space(state: &AppState) -> String { + let qm = &state.queue_manager; + qm.pause_all(); + let mut job = nzb_parser::parse_nzb("held", NZB.as_bytes()).expect("parse nzb"); + job.work_dir = qm.incomplete_dir().join(&job.id); + job.output_dir = qm + .output_dir_for(&job.category, &job.name) + .expect("output dir"); + let id = job.id.clone(); + qm.add_job(job, Some(NZB.as_bytes().to_vec())) + .expect("add job"); + qm.resume_all(); + // No volume can ever have u64::MAX bytes free. + qm.set_min_free_space(u64::MAX); + qm.enforce_disk_guard(); + assert!(qm.is_paused()); + id +} + +async fn sab(state: Arc, mode: &str) -> serde_json::Value { + let response = h_sabnzbd_api_get( + State(state), + Query(SabApiRequest { + mode: Some(mode.to_string()), + ..Default::default() + }), + ) + .await + .unwrap() + .into_response(); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + serde_json::from_slice(&body).unwrap() +} + +async fn status_json(state: Arc) -> serde_json::Value { + #[cfg(feature = "webdav")] + let status = h_status(State(state), axum::Extension(None)).await; + #[cfg(not(feature = "webdav"))] + let status = h_status(State(state)).await; + serde_json::to_value(status.ok().unwrap().0).unwrap() +} + +#[tokio::test] +async fn native_queue_and_status_report_disk_space_pause_reason() { + let (state, _tempdir) = test_state(); + let id = hold_for_disk_space(&state); + + let queue = h_queue_list(State(state.clone()), Query(QueueQuery::default())) + .await + .ok() + .unwrap() + .0; + let queue = serde_json::to_value(queue).unwrap(); + assert_eq!(queue["paused"], true); + assert_eq!(queue["pause_reason"], "disk_space"); + let job = &queue["jobs"][0]; + assert_eq!(job["id"], id.as_str()); + assert_eq!(job["status"], "paused"); + assert_eq!(job["pause_reason"], "disk_space"); + + let status = status_json(state.clone()).await; + assert_eq!(status["paused"], true); + assert_eq!(status["pause_reason"], "disk_space"); + + // Once space frees up the hold lifts and the reason clears. + state.queue_manager.set_min_free_space(1); + state.queue_manager.enforce_disk_guard(); + let status = status_json(state.clone()).await; + assert_eq!(status["paused"], false); + assert!(status["pause_reason"].is_null()); +} + +#[tokio::test] +async fn native_queue_reports_user_pause_as_global() { + let (state, _tempdir) = test_state(); + state.queue_manager.pause_all(); + let queue = h_queue_list(State(state.clone()), Query(QueueQuery::default())) + .await + .ok() + .unwrap() + .0; + let queue = serde_json::to_value(queue).unwrap(); + assert_eq!(queue["pause_reason"], "global"); +} + +#[tokio::test] +async fn sab_reports_a_disk_space_warning_while_the_guard_holds_downloads() { + let (state, _tempdir) = test_state(); + + let queue = sab(state.clone(), "queue").await; + assert_eq!(queue["queue"]["have_warnings"], "0"); + + hold_for_disk_space(&state); + + let queue = sab(state.clone(), "queue").await; + assert_eq!(queue["queue"]["status"], "Paused"); + assert_eq!(queue["queue"]["paused"], true); + assert_eq!(queue["queue"]["have_warnings"], "1"); + + let full = sab(state.clone(), "fullstatus").await; + assert_eq!(full["status"]["have_warnings"], "1"); + let warning = &full["status"]["warnings"][0]; + assert_eq!(warning["type"], "WARNING"); + assert!(warning["text"].as_str().unwrap().contains("diskspace")); + assert!(warning["time"].is_i64()); + + let warnings = sab(state.clone(), "warnings").await; + assert_eq!(warnings["warnings"][0], *warning); + + state.queue_manager.set_min_free_space(1); + state.queue_manager.enforce_disk_guard(); + let full = sab(state, "fullstatus").await; + assert_eq!(full["status"]["have_warnings"], "0"); + assert_eq!(full["status"]["warnings"], serde_json::json!([])); +} diff --git a/apps/rustnzb/tests/e2e_postproc_detection.rs b/apps/rustnzb/tests/e2e_postproc_detection.rs index b43a8625..4bccce7b 100644 --- a/apps/rustnzb/tests/e2e_postproc_detection.rs +++ b/apps/rustnzb/tests/e2e_postproc_detection.rs @@ -157,6 +157,7 @@ async fn normal_subjects_pipeline_finds_files() { // With zero failures, par2 should be skipped (files known-good) let config_zero = PostProcConfig { cleanup_after_extract: false, + delete_archives: true, output_dir: None, articles_failed: 0, content_articles_failed: 0, @@ -184,6 +185,7 @@ async fn normal_subjects_pipeline_finds_files() { // but the stage should NOT be skipped — confirming detection works) let config_fail = PostProcConfig { cleanup_after_extract: false, + delete_archives: true, output_dir: None, articles_failed: 1, content_articles_failed: 1, @@ -319,6 +321,7 @@ async fn obfuscated_pipeline_skips_everything() { let dir = make_work_dir(&obfuscated_files); let config = PostProcConfig { cleanup_after_extract: false, + delete_archives: true, output_dir: None, articles_failed: 0, content_articles_failed: 0, @@ -512,6 +515,7 @@ async fn deobfuscation_enables_full_pipeline() { // Before deobfuscation: pipeline skips everything let config = PostProcConfig { cleanup_after_extract: false, + delete_archives: true, output_dir: None, articles_failed: 0, content_articles_failed: 0, @@ -535,6 +539,7 @@ async fn deobfuscation_enables_full_pipeline() { // run with articles_failed > 0 so repair is attempted. let config_fail = PostProcConfig { cleanup_after_extract: false, + delete_archives: true, output_dir: None, articles_failed: 1, content_articles_failed: 1, @@ -701,6 +706,7 @@ async fn pipeline_with_only_archives_no_par2() { let output_dir = tempfile::tempdir().unwrap(); let config = PostProcConfig { cleanup_after_extract: false, + delete_archives: true, output_dir: Some(output_dir.path().to_path_buf()), articles_failed: 0, content_articles_failed: 0, diff --git a/apps/rustnzb/tests/history_retry_idempotent.rs b/apps/rustnzb/tests/history_retry_idempotent.rs new file mode 100644 index 00000000..6efff013 --- /dev/null +++ b/apps/rustnzb/tests/history_retry_idempotent.rs @@ -0,0 +1,89 @@ +//! BUG-64: `POST /api/history/{id}/retry` is idempotent per history entry. + +mod support; + +use chrono::Utc; +use nzb_web::nzb_core::models::{HistoryEntry, JobFailureCode, JobStatus}; +use reqwest::StatusCode; +use support::{sample_nzb_bytes, start_test_server}; + +async fn setup_auth(client: &reqwest::Client, base_url: &str) -> String { + let setup = client + .post(format!("{base_url}/api/auth/setup")) + .json(&serde_json::json!({ + "username": "retry-test", + "password": "retry-test-password" + })) + .send() + .await + .expect("auth setup failed"); + assert_eq!(setup.status(), StatusCode::OK); + setup.json::().await.unwrap()["access_token"] + .as_str() + .expect("auth setup should return an access token") + .to_string() +} + +fn failed_history_entry(id: &str, complete_dir: &std::path::Path) -> HistoryEntry { + HistoryEntry { + id: id.into(), + name: "Retry Me".into(), + category: "Default".into(), + status: JobStatus::Failed, + total_bytes: 10, + downloaded_bytes: 5, + added_at: Utc::now(), + completed_at: Utc::now(), + download_time_secs: None, + output_dir: complete_dir.join("Retry Me"), + stages: Vec::new(), + error_message: Some("missing articles".into()), + failure_code: Some(JobFailureCode::ArticlesUnavailable), + post_processing: None, + delete_archives: None, + server_stats: Vec::new(), + nzb_data: Some(sample_nzb_bytes()), + retry_data: None, + } +} + +#[tokio::test] +async fn concurrent_history_retries_enqueue_one_job() { + let app = start_test_server(Vec::new()).await; + let client = reqwest::Client::new(); + let access = setup_auth(&client, &app.base_url).await; + client + .post(format!("{}/api/queue/pause", app.base_url)) + .bearer_auth(&access) + .send() + .await + .unwrap() + .error_for_status() + .unwrap(); + let entry = failed_history_entry("failed-history-entry", &app.complete_dir); + app.state + .queue_manager + .with_db(|db| db.history_insert(&entry).expect("insert history")); + + let url = format!("{}/api/history/failed-history-entry/retry", app.base_url); + let mut requests = tokio::task::JoinSet::new(); + for _ in 0..5 { + requests.spawn(client.post(&url).bearer_auth(&access).send()); + } + + let mut job_ids = Vec::new(); + while let Some(response) = requests.join_next().await { + let response = response.unwrap().unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = response.json::().await.unwrap(); + assert_eq!(body["status"], true, "{body}"); + job_ids.push(body["nzo_ids"][0].as_str().unwrap().to_string()); + } + + let jobs = app.state.queue_manager.get_jobs(); + assert_eq!(jobs.len(), 1, "five retries must enqueue one job"); + assert!( + job_ids.iter().all(|id| id == &jobs[0].id), + "every caller gets the one retried job id: {job_ids:?}" + ); +} diff --git a/apps/rustnzb/tests/inline_job_password.rs b/apps/rustnzb/tests/inline_job_password.rs new file mode 100644 index 00000000..a1bf589a --- /dev/null +++ b/apps/rustnzb/tests/inline_job_password.rs @@ -0,0 +1,183 @@ +//! The native add endpoints honour SABnzbd's inline job-password convention +//! (`name{{password}}`, `name/password`) and an explicit `password` field. +//! Passwords are read back through the SABnzbd queue, whose slot contract +//! includes the job password. + +mod support; + +use reqwest::StatusCode; +use support::{sample_nzb_bytes, start_test_server}; + +const META_PASSWORD_NZB: &str = r#" + + meta-pw + + alt.binaries.test + + article1@example.com + + +"#; + +struct Harness { + client: reqwest::Client, + base_url: String, + access: String, + sab_key: String, + _app: support::TestApp, +} + +impl Harness { + async fn start() -> Self { + let app = start_test_server(Vec::new()).await; + let client = reqwest::Client::new(); + let base_url = app.base_url.clone(); + let setup = client + .post(format!("{base_url}/api/auth/setup")) + .json(&serde_json::json!({ + "username": "inline-pw-test", + "password": "inline-pw-test-password" + })) + .send() + .await + .unwrap(); + assert_eq!(setup.status(), StatusCode::OK); + let access = setup.json::().await.unwrap()["access_token"] + .as_str() + .unwrap() + .to_string(); + let sab_key = client + .post(format!("{base_url}/api/config/sabnzbd-api-key/rotate")) + .bearer_auth(&access) + .json(&serde_json::json!({})) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json::() + .await + .unwrap()["api_key"] + .as_str() + .unwrap() + .to_string(); + client + .post(format!("{base_url}/api/queue/pause")) + .bearer_auth(&access) + .send() + .await + .unwrap() + .error_for_status() + .unwrap(); + Self { + client, + base_url, + access, + sab_key, + _app: app, + } + } + + async fn upload(&self, query: &str, file_name: &str, nzb: Vec, password: Option<&str>) { + let part = reqwest::multipart::Part::bytes(nzb) + .file_name(file_name.to_string()) + .mime_str("application/x-nzb") + .unwrap(); + let mut form = reqwest::multipart::Form::new().part("file", part); + if let Some(password) = password { + form = form.text("password", password.to_string()); + } + let response = self + .client + .post(format!("{}/api/queue/add{query}", self.base_url)) + .bearer_auth(&self.access) + .multipart(form) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.text().await.unwrap(); + assert_eq!(status, StatusCode::OK, "{file_name}: {body}"); + } + + /// (job name, password) of the only queued job, via the SABnzbd queue. + async fn only_job(&self) -> (String, String) { + let queue = self + .client + .get(format!( + "{}/sabnzbd/api?mode=queue&output=json&apikey={}", + self.base_url, self.sab_key + )) + .send() + .await + .unwrap() + .json::() + .await + .unwrap(); + let slots = queue["queue"]["slots"].as_array().expect("slots"); + assert_eq!(slots.len(), 1, "{queue}"); + ( + slots[0]["filename"].as_str().unwrap().to_string(), + slots[0]["password"].as_str().unwrap().to_string(), + ) + } +} + +#[tokio::test] +async fn native_add_takes_inline_password_from_file_name() { + let h = Harness::start().await; + h.upload("", "My.Release{{filepw}}.nzb", sample_nzb_bytes(), None) + .await; + assert_eq!( + h.only_job().await, + ("My.Release".to_string(), "filepw".to_string()) + ); +} + +#[tokio::test] +async fn native_add_takes_inline_password_from_name_override() { + let h = Harness::start().await; + h.upload( + "?name=Chosen%2Fnamepw", + "upload.nzb", + sample_nzb_bytes(), + None, + ) + .await; + assert_eq!( + h.only_job().await, + ("Chosen".to_string(), "namepw".to_string()) + ); +} + +#[tokio::test] +async fn native_add_explicit_password_field_wins() { + let h = Harness::start().await; + h.upload( + "", + "Show{{inline-pw}}.nzb", + META_PASSWORD_NZB.as_bytes().to_vec(), + Some("explicit-pw"), + ) + .await; + assert_eq!( + h.only_job().await, + ("Show".to_string(), "explicit-pw".to_string()) + ); +} + +#[tokio::test] +async fn native_add_inline_password_beats_nzb_meta() { + let h = Harness::start().await; + h.upload( + "", + "Show{{inline-pw}}.nzb", + META_PASSWORD_NZB.as_bytes().to_vec(), + None, + ) + .await; + assert_eq!( + h.only_job().await, + ("Show".to_string(), "inline-pw".to_string()) + ); +} diff --git a/apps/rustnzb/tests/native_add_form_fields.rs b/apps/rustnzb/tests/native_add_form_fields.rs new file mode 100644 index 00000000..9e13f2ad --- /dev/null +++ b/apps/rustnzb/tests/native_add_form_fields.rs @@ -0,0 +1,197 @@ +//! `POST /api/queue/add` accepts its options as multipart text fields as well +//! as query parameters (the README's `-F "category=tv"` example). Only parts +//! that carry a file name are NZB uploads; form fields override the query. + +mod support; + +use reqwest::StatusCode; +use reqwest::multipart::{Form, Part}; +use support::{sample_nzb_bytes, start_test_server}; + +struct Harness { + client: reqwest::Client, + base_url: String, + access: String, + _app: support::TestApp, +} + +impl Harness { + async fn start() -> Self { + let app = start_test_server(Vec::new()).await; + let client = reqwest::Client::new(); + let base_url = app.base_url.clone(); + let setup = client + .post(format!("{base_url}/api/auth/setup")) + .json(&serde_json::json!({ + "username": "form-fields-test", + "password": "form-fields-test-password" + })) + .send() + .await + .unwrap(); + assert_eq!(setup.status(), StatusCode::OK); + let access = setup.json::().await.unwrap()["access_token"] + .as_str() + .unwrap() + .to_string(); + client + .post(format!("{base_url}/api/queue/pause")) + .bearer_auth(&access) + .send() + .await + .unwrap() + .error_for_status() + .unwrap(); + Self { + client, + base_url, + access, + _app: app, + } + } + + fn nzb_form() -> Form { + Form::new().part( + "file", + Part::bytes(sample_nzb_bytes()) + .file_name("upload.nzb") + .mime_str("application/x-nzb") + .unwrap(), + ) + } + + async fn add(&self, query: &str, form: Form) -> (StatusCode, serde_json::Value) { + let response = self + .client + .post(format!("{}/api/queue/add{query}", self.base_url)) + .bearer_auth(&self.access) + .multipart(form) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.json().await.unwrap_or(serde_json::Value::Null); + (status, body) + } + + async fn jobs(&self) -> Vec { + self.client + .get(format!("{}/api/queue", self.base_url)) + .bearer_auth(&self.access) + .send() + .await + .unwrap() + .json::() + .await + .unwrap()["jobs"] + .as_array() + .expect("jobs") + .clone() + } + + /// (name, category, priority) of the only queued job. + async fn only_job(&self) -> (String, String, u64) { + let jobs = self.jobs().await; + assert_eq!(jobs.len(), 1, "{jobs:?}"); + ( + jobs[0]["name"].as_str().unwrap().to_string(), + jobs[0]["category"].as_str().unwrap().to_string(), + jobs[0]["priority"].as_u64().unwrap(), + ) + } +} + +#[tokio::test] +async fn readme_category_form_field_is_applied() { + let h = Harness::start().await; + let (status, body) = h.add("", Harness::nzb_form().text("category", "tv")).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["nzo_ids"].as_array().unwrap().len(), 1, "{body}"); + assert_eq!( + h.only_job().await, + ("upload".to_string(), "tv".to_string(), 1) + ); +} + +#[tokio::test] +async fn text_fields_before_the_file_are_options_not_uploads() { + let h = Harness::start().await; + let form = Form::new() + .text("cat", "movies") + .text("priority", "2") + .text("nzbname", "Chosen.Name") + .text("unrelated", "ignored") + .part( + "file", + Part::bytes(sample_nzb_bytes()).file_name("upload.nzb"), + ); + let (status, body) = h.add("", form).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + h.only_job().await, + ("Chosen.Name".to_string(), "movies".to_string(), 2) + ); +} + +#[tokio::test] +async fn name_form_field_overrides_job_name() { + let h = Harness::start().await; + let (status, body) = h.add("", Harness::nzb_form().text("name", "Named")).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(h.only_job().await.0, "Named"); +} + +#[tokio::test] +async fn query_params_still_apply() { + let h = Harness::start().await; + let (status, body) = h + .add( + "?category=tv&priority=0&name=FromQuery", + Harness::nzb_form(), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + h.only_job().await, + ("FromQuery".to_string(), "tv".to_string(), 0) + ); +} + +#[tokio::test] +async fn form_fields_override_query_params() { + let h = Harness::start().await; + let form = Harness::nzb_form() + .text("category", "movies") + .text("priority", "3") + .text("name", "FromForm"); + let (status, body) = h.add("?category=tv&priority=0&name=FromQuery", form).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + h.only_job().await, + ("FromForm".to_string(), "movies".to_string(), 3) + ); +} + +#[tokio::test] +async fn invalid_priority_form_field_is_rejected() { + let h = Harness::start().await; + let (status, body) = h + .add("", Harness::nzb_form().text("priority", "urgent")) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{body}"); + assert!(h.jobs().await.is_empty()); +} + +#[tokio::test] +async fn bare_file_field_without_file_name_is_still_an_upload() { + let h = Harness::start().await; + let form = Form::new() + .text("file", String::from_utf8(sample_nzb_bytes()).unwrap()) + .text("category", "tv"); + let (status, body) = h.add("", form).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + h.only_job().await, + ("unknown".to_string(), "tv".to_string(), 1) + ); +} diff --git a/apps/rustnzb/tests/queue_bulk_api.rs b/apps/rustnzb/tests/queue_bulk_api.rs new file mode 100644 index 00000000..93d8a85f --- /dev/null +++ b/apps/rustnzb/tests/queue_bulk_api.rs @@ -0,0 +1,218 @@ +//! BUG-63: `POST /api/queue/bulk` counts an id that matches no job as a +//! failure, and reports which ids failed and why. + +use std::sync::Arc; + +use arc_swap::ArcSwap; +use chrono::Utc; +use nzb_web::auth::{CredentialStore, StoredCredentials, TokenStore}; +use nzb_web::nzb_core::config::AppConfig; +use nzb_web::nzb_core::db::Database; +use nzb_web::nzb_core::models::{JobStatus, NzbJob, Priority}; +use nzb_web::{AppState, LogBuffer, QueueManager}; +use rustnzb::server::build_router; + +struct App { + base: String, + state: Arc, + _temp: tempfile::TempDir, + handle: tokio::task::JoinHandle<()>, +} + +impl Drop for App { + fn drop(&mut self) { + self.handle.abort(); + } +} + +async fn start_app() -> App { + let temp = tempfile::tempdir().unwrap(); + let config = AppConfig::default(); + let logs = LogBuffer::new(); + let queue = QueueManager::new( + Vec::new(), + Database::open_memory().unwrap(), + temp.path().join("incomplete"), + temp.path().join("complete"), + logs.clone(), + config.general.max_active_downloads, + config.categories.clone(), + config.general.min_free_space_bytes, + config.general.speed_limit_bps, + false, + config.general.max_nested_archive_depth, + config.general.abort_hopeless, + config.general.early_failure_check, + config.general.required_completion_pct, + config.general.article_timeout_secs, + ); + // Keep every job queued: no servers are configured. + queue.pause_all(); + let credentials = Arc::new(CredentialStore::new(temp.path().to_path_buf())); + credentials + .set_credentials(StoredCredentials { + username: "admin".into(), + password: "password".into(), + }) + .unwrap(); + let state = Arc::new(AppState::new( + Arc::new(ArcSwap::from_pointee(config)), + temp.path().join("config.toml"), + queue, + logs, + Arc::new(TokenStore::new()), + credentials, + )); + let router = build_router(state.clone()); + #[cfg(feature = "webdav")] + let router = router.layer(axum::Extension(None::>)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let handle = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + + let root = temp.path().to_path_buf(); + for id in ["job-a", "job-b"] { + let job = NzbJob { + id: id.into(), + name: id.into(), + category: "Default".into(), + status: JobStatus::Queued, + priority: Priority::Normal, + total_bytes: 100, + downloaded_bytes: 0, + file_count: 0, + files_completed: 0, + article_count: 0, + articles_downloaded: 0, + articles_failed: 0, + added_at: Utc::now(), + completed_at: None, + work_dir: root.join("incomplete").join(id), + output_dir: root.join("complete").join(id), + password: None, + error_message: None, + speed_bps: 0, + server_stats: Vec::new(), + pp_override: None, + files: Vec::new(), + delete_archives: None, + }; + state.queue_manager.add_job(job, None).unwrap(); + } + + App { + base, + state, + _temp: temp, + handle, + } +} + +async fn login(app: &App, client: &reqwest::Client) -> String { + let tokens = client + .post(format!("{}/api/auth/login", app.base)) + .json(&serde_json::json!({"username": "admin", "password": "password"})) + .send() + .await + .unwrap() + .json::() + .await + .unwrap(); + tokens["access_token"].as_str().unwrap().to_string() +} + +fn job_ids(app: &App) -> Vec { + app.state + .queue_manager + .get_jobs() + .into_iter() + .map(|job| job.id) + .collect() +} + +const MISSING: &str = "00000000-0000-0000-0000-000000000000"; + +async fn bulk( + app: &App, + client: &reqwest::Client, + access: &str, + body: serde_json::Value, +) -> serde_json::Value { + let response = client + .post(format!("{}/api/queue/bulk", app.base)) + .bearer_auth(access) + .json(&body) + .send() + .await + .unwrap(); + assert_eq!(response.status().as_u16(), 200); + response.json().await.unwrap() +} + +#[tokio::test] +async fn bulk_delete_counts_unknown_ids_as_failed() { + let app = start_app().await; + let client = reqwest::Client::new(); + let access = login(&app, &client).await; + + let response = bulk( + &app, + &client, + &access, + serde_json::json!({"action": "delete", "ids": ["job-a", "job-b", MISSING]}), + ) + .await; + assert_eq!(response["status"], false, "{response}"); + assert_eq!(response["succeeded"], 2, "{response}"); + assert_eq!(response["failed"], 1, "{response}"); + let failures = response["failures"].as_array().expect("failures array"); + assert_eq!(failures.len(), 1, "{response}"); + assert_eq!(failures[0]["id"], MISSING); + assert_eq!(failures[0]["error"]["error_kind"], "job_not_found"); + assert!(job_ids(&app).is_empty()); +} + +#[tokio::test] +async fn every_bulk_action_counts_unknown_ids_as_failed() { + let app = start_app().await; + let client = reqwest::Client::new(); + let access = login(&app, &client).await; + + for (body, succeeded) in [ + (serde_json::json!({"action": "pause"}), 2), + (serde_json::json!({"action": "priority", "value": 2}), 2), + (serde_json::json!({"action": "category", "value": "tv"}), 2), + // Individual resume is refused while downloads are globally paused, + // so only the unknown id's outcome is checked here. + (serde_json::json!({"action": "resume"}), 0), + ] { + let mut body = body; + body["ids"] = serde_json::json!(["job-a", "job-b", MISSING]); + let response = bulk(&app, &client, &access, body.clone()).await; + assert_eq!(response["status"], false, "{body}: {response}"); + assert_eq!(response["succeeded"], succeeded, "{body}: {response}"); + assert_eq!(response["failed"], 3 - succeeded, "{body}: {response}"); + let failures = response["failures"].as_array().expect("failures array"); + let missing = failures + .iter() + .find(|failure| failure["id"] == MISSING) + .unwrap_or_else(|| panic!("{body}: {response}")); + if succeeded == 2 { + assert_eq!(missing["error"]["error_kind"], "job_not_found", "{body}"); + } + } + + // A fully successful request reports no failures. + let response = bulk( + &app, + &client, + &access, + serde_json::json!({"action": "pause", "ids": ["job-a"]}), + ) + .await; + assert_eq!(response["status"], true, "{response}"); + assert_eq!(response["succeeded"], 1); + assert_eq!(response["failures"], serde_json::json!([])); +} diff --git a/apps/rustnzb/tests/queue_password_mask.rs b/apps/rustnzb/tests/queue_password_mask.rs new file mode 100644 index 00000000..fab380da --- /dev/null +++ b/apps/rustnzb/tests/queue_password_mask.rs @@ -0,0 +1,130 @@ +//! The native queue API masks the job archive password the way the config +//! API masks NNTP server passwords, while the SABnzbd queue keeps returning +//! it (the SAB slot contract includes `password`). + +mod support; + +use reqwest::StatusCode; +use support::{sample_nzb_bytes, start_test_server}; + +const META_PASSWORD_NZB: &str = r#" + + archive-secret + + alt.binaries.test + + article1@example.com + + +"#; + +async fn upload(client: &reqwest::Client, base_url: &str, access: &str, name: &str, nzb: Vec) { + let part = reqwest::multipart::Part::bytes(nzb) + .file_name(name.to_string()) + .mime_str("application/x-nzb") + .unwrap(); + let response = client + .post(format!("{base_url}/api/queue/add")) + .bearer_auth(access) + .multipart(reqwest::multipart::Form::new().part("file", part)) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn native_queue_masks_archive_password_but_sab_queue_keeps_it() { + let app = start_test_server(Vec::new()).await; + let base_url = app.base_url.clone(); + let client = reqwest::Client::new(); + let access = client + .post(format!("{base_url}/api/auth/setup")) + .json(&serde_json::json!({ + "username": "mask-test", + "password": "mask-test-password" + })) + .send() + .await + .unwrap() + .json::() + .await + .unwrap()["access_token"] + .as_str() + .unwrap() + .to_string(); + client + .post(format!("{base_url}/api/queue/pause")) + .bearer_auth(&access) + .send() + .await + .unwrap() + .error_for_status() + .unwrap(); + + upload( + &client, + &base_url, + &access, + "Secret.nzb", + META_PASSWORD_NZB.as_bytes().to_vec(), + ) + .await; + upload(&client, &base_url, &access, "Open.nzb", sample_nzb_bytes()).await; + + let response = client + .get(format!("{base_url}/api/queue")) + .bearer_auth(&access) + .send() + .await + .unwrap(); + let raw = response.text().await.unwrap(); + assert!( + !raw.contains("archive-secret"), + "native queue leaks the archive password: {raw}" + ); + let queue: serde_json::Value = serde_json::from_str(&raw).unwrap(); + let password_of = |name: &str| { + queue["jobs"] + .as_array() + .unwrap() + .iter() + .find(|job| job["name"] == name) + .unwrap_or_else(|| panic!("{name} not queued: {queue}"))["password"] + .clone() + }; + assert_eq!(password_of("Secret"), "********"); + assert_eq!(password_of("Open"), ""); + + // The SABnzbd queue still returns the real password. + let sab_key = client + .post(format!("{base_url}/api/config/sabnzbd-api-key/rotate")) + .bearer_auth(&access) + .json(&serde_json::json!({})) + .send() + .await + .unwrap() + .json::() + .await + .unwrap()["api_key"] + .as_str() + .unwrap() + .to_string(); + let sab = client + .get(format!( + "{base_url}/sabnzbd/api?mode=queue&output=json&apikey={sab_key}" + )) + .send() + .await + .unwrap() + .json::() + .await + .unwrap(); + let slot = sab["queue"]["slots"] + .as_array() + .unwrap() + .iter() + .find(|slot| slot["filename"] == "Secret") + .unwrap_or_else(|| panic!("Secret not in SAB queue: {sab}")); + assert_eq!(slot["password"], "archive-secret"); +} diff --git a/apps/rustnzb/tests/queue_sort_api.rs b/apps/rustnzb/tests/queue_sort_api.rs new file mode 100644 index 00000000..bacfacb9 --- /dev/null +++ b/apps/rustnzb/tests/queue_sort_api.rs @@ -0,0 +1,270 @@ +//! BUG-62: `POST /api/queue/sort` and SABnzbd `mode=queue&name=sort` honour +//! the requested sort field and direction instead of always sorting by +//! remaining percentage. + +use std::sync::Arc; + +use arc_swap::ArcSwap; +use chrono::{Duration, Utc}; +use nzb_web::auth::{CredentialStore, StoredCredentials, TokenStore}; +use nzb_web::nzb_core::config::AppConfig; +use nzb_web::nzb_core::db::Database; +use nzb_web::nzb_core::models::{JobStatus, NzbJob, Priority}; +use nzb_web::{AppState, LogBuffer, QueueManager}; +use rustnzb::server::build_router; + +const API_KEY: &str = "sort-test-key"; + +struct App { + base: String, + state: Arc, + _temp: tempfile::TempDir, + handle: tokio::task::JoinHandle<()>, +} + +impl Drop for App { + fn drop(&mut self) { + self.handle.abort(); + } +} + +async fn start_app() -> App { + let temp = tempfile::tempdir().unwrap(); + let mut config = AppConfig::default(); + config.general.api_key = Some(API_KEY.into()); + let logs = LogBuffer::new(); + let queue = QueueManager::new( + Vec::new(), + Database::open_memory().unwrap(), + temp.path().join("incomplete"), + temp.path().join("complete"), + logs.clone(), + config.general.max_active_downloads, + config.categories.clone(), + config.general.min_free_space_bytes, + config.general.speed_limit_bps, + false, + config.general.max_nested_archive_depth, + config.general.abort_hopeless, + config.general.early_failure_check, + config.general.required_completion_pct, + config.general.article_timeout_secs, + ); + // Keep every job queued: the test only exercises ordering. + queue.pause_all(); + let credentials = Arc::new(CredentialStore::new(temp.path().to_path_buf())); + credentials + .set_credentials(StoredCredentials { + username: "admin".into(), + password: "password".into(), + }) + .unwrap(); + let state = Arc::new(AppState::new( + Arc::new(ArcSwap::from_pointee(config)), + temp.path().join("config.toml"), + queue, + logs, + Arc::new(TokenStore::new()), + credentials, + )); + let router = build_router(state.clone()); + #[cfg(feature = "webdav")] + let router = router.layer(axum::Extension(None::>)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let handle = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + + let root = temp.path().to_path_buf(); + let now = Utc::now(); + // (id, name, total, downloaded, priority, minutes old) + for (id, name, total, downloaded, priority, age) in [ + ("job-a", "charlie", 300u64, 0u64, Priority::Normal, 10i64), + ("job-b", "Alpha", 100, 90, Priority::High, 30), + ("job-c", "bravo", 200, 20, Priority::Normal, 20), + ] { + let job = NzbJob { + id: id.into(), + name: name.into(), + category: "Default".into(), + status: JobStatus::Queued, + priority, + total_bytes: total, + downloaded_bytes: downloaded, + file_count: 0, + files_completed: 0, + article_count: 0, + articles_downloaded: 0, + articles_failed: 0, + added_at: now - Duration::minutes(age), + completed_at: None, + work_dir: root.join("incomplete").join(id), + output_dir: root.join("complete").join(id), + password: None, + error_message: None, + speed_bps: 0, + server_stats: Vec::new(), + pp_override: None, + files: Vec::new(), + delete_archives: None, + }; + state.queue_manager.add_job(job, None).unwrap(); + } + + App { + base, + state, + _temp: temp, + handle, + } +} + +async fn login(app: &App, client: &reqwest::Client) -> String { + let tokens = client + .post(format!("{}/api/auth/login", app.base)) + .json(&serde_json::json!({"username": "admin", "password": "password"})) + .send() + .await + .unwrap() + .json::() + .await + .unwrap(); + tokens["access_token"].as_str().unwrap().to_string() +} + +fn order(app: &App) -> Vec { + app.state + .queue_manager + .get_jobs() + .into_iter() + .map(|job| job.id) + .collect() +} + +async fn sort( + app: &App, + client: &reqwest::Client, + access: &str, + body: serde_json::Value, +) -> (u16, serde_json::Value) { + let response = client + .post(format!("{}/api/queue/sort", app.base)) + .bearer_auth(access) + .json(&body) + .send() + .await + .unwrap(); + let status = response.status().as_u16(); + let text = response.text().await.unwrap(); + ( + status, + serde_json::from_str(&text).unwrap_or(serde_json::Value::Null), + ) +} + +#[tokio::test] +async fn native_sort_honours_sort_by_and_direction() { + let app = start_app().await; + let client = reqwest::Client::new(); + let access = login(&app, &client).await; + + for (body, expected) in [ + ( + serde_json::json!({"sort_by": "name", "direction": "asc"}), + ["job-b", "job-c", "job-a"], + ), + ( + serde_json::json!({"sort_by": "name", "direction": "desc"}), + ["job-a", "job-c", "job-b"], + ), + ( + serde_json::json!({"sort_by": "size"}), + ["job-b", "job-c", "job-a"], + ), + ( + serde_json::json!({"sort_by": "size", "ascending": false}), + ["job-a", "job-c", "job-b"], + ), + // `direction` wins over `ascending` when both are sent. + ( + serde_json::json!({"sort_by": "size", "ascending": false, "direction": "asc"}), + ["job-b", "job-c", "job-a"], + ), + ( + serde_json::json!({"sort_by": "age", "direction": "asc"}), + ["job-a", "job-c", "job-b"], + ), + ( + serde_json::json!({"sort_by": "priority", "direction": "desc"}), + ["job-b", "job-a", "job-c"], + ), + // Legacy body: remaining percentage, ascending by default. + (serde_json::json!({}), ["job-b", "job-c", "job-a"]), + ( + serde_json::json!({"ascending": false}), + ["job-a", "job-c", "job-b"], + ), + ] { + let (status, response) = sort(&app, &client, &access, body.clone()).await; + assert_eq!(status, 200, "{body}: {response}"); + assert_eq!(response["status"], true, "{body}"); + assert_eq!(order(&app), expected, "{body}"); + } +} + +#[tokio::test] +async fn native_sort_rejects_unknown_field_or_direction() { + let app = start_app().await; + let client = reqwest::Client::new(); + let access = login(&app, &client).await; + let before = order(&app); + + for body in [ + serde_json::json!({"sort_by": "colour"}), + serde_json::json!({"sort_by": "name", "direction": "sideways"}), + ] { + let (status, response) = sort(&app, &client, &access, body.clone()).await; + assert_eq!(status, 400, "{body}: {response}"); + assert_eq!(response["error_kind"], "bad_request", "{body}"); + assert_eq!(order(&app), before, "{body} must not reorder the queue"); + } +} + +async fn sab(app: &App, client: &reqwest::Client, query: &str) -> serde_json::Value { + client + .get(format!( + "{}/api?mode=queue&name=sort&output=json&apikey={API_KEY}&{query}", + app.base + )) + .send() + .await + .unwrap() + .json() + .await + .unwrap() +} + +#[tokio::test] +async fn sab_queue_sort_honours_sort_and_dir() { + let app = start_app().await; + let client = reqwest::Client::new(); + + for (query, expected) in [ + ("sort=name&dir=asc", ["job-b", "job-c", "job-a"]), + ("sort=name&dir=desc", ["job-a", "job-c", "job-b"]), + ("sort=size&dir=desc", ["job-a", "job-c", "job-b"]), + ("sort=avg_age&dir=asc", ["job-a", "job-c", "job-b"]), + ("sort=remaining&dir=asc", ["job-b", "job-c", "job-a"]), + ("sort=remaining&dir=desc", ["job-a", "job-c", "job-b"]), + ] { + let response = sab(&app, &client, query).await; + assert_eq!(response["status"], true, "{query}: {response}"); + assert_eq!(order(&app), expected, "{query}"); + } + + let before = order(&app); + let response = sab(&app, &client, "sort=colour&dir=asc").await; + assert_eq!(response["status"], false, "{response}"); + assert_eq!(order(&app), before); +} diff --git a/apps/rustnzb/tests/sab_api_key_api.rs b/apps/rustnzb/tests/sab_api_key_api.rs index 6710e092..b2a11b46 100644 --- a/apps/rustnzb/tests/sab_api_key_api.rs +++ b/apps/rustnzb/tests/sab_api_key_api.rs @@ -51,8 +51,6 @@ async fn start_test_server() -> ( credential_store, )); let router = build_router(state.clone()); - #[cfg(feature = "webdav")] - let router = router.layer(axum::Extension(None::>)); let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await diff --git a/apps/rustnzb/tests/sabnzbd_import_apply_test.rs b/apps/rustnzb/tests/sabnzbd_import_apply_test.rs index 6cce1645..4b00a278 100644 --- a/apps/rustnzb/tests/sabnzbd_import_apply_test.rs +++ b/apps/rustnzb/tests/sabnzbd_import_apply_test.rs @@ -125,3 +125,73 @@ async fn apply_accepts_absolute_dirs() { std::path::PathBuf::from("/downloads/incomplete") ); } + +fn preview_with_categories(categories: serde_json::Value) -> SabnzbdImportPreview { + serde_json::from_value(serde_json::json!({ + "servers": [], + "categories": categories, + "general": { + "api_key": null, + "complete_dir": null, + "incomplete_dir": null, + "speed_limit_bps": 0 + }, + "rss_feeds": [], + "warnings": [], + "skipped_fields": [] + })) + .expect("preview JSON should deserialize") +} + +/// Negative: imported categories get the same validation as a category +/// save; an unsafe name or output_dir rejects the whole apply with 400, +/// names every offending entry, and changes nothing. +#[tokio::test] +async fn apply_rejects_unsafe_imported_categories() { + let (state, _tempdir) = test_state(); + let before = state.config().categories.clone(); + let preview = preview_with_categories(serde_json::json!([ + {"name": "tv", "output_dir": "shows/tv", "post_processing": 3}, + {"name": "../escape", "output_dir": null, "post_processing": 3}, + {"name": "movies", "output_dir": "../../etc", "post_processing": 3}, + ])); + + let error = match h_setup_apply(State(state.clone()), Json(preview)).await { + Ok(_) => panic!("expected unsafe categories to be rejected"), + Err(e) => e, + }; + + assert_eq!(error.status(), StatusCode::BAD_REQUEST); + let message = error.to_string(); + assert!(message.contains("../escape"), "{message}"); + assert!(message.contains("movies"), "{message}"); + assert!(!message.contains("'tv'"), "{message}"); + let after = state.config().categories.clone(); + assert_eq!( + after.iter().map(|c| &c.name).collect::>(), + before.iter().map(|c| &c.name).collect::>(), + "rejected apply must not have replaced the categories" + ); +} + +/// Positive: safe imported categories replace the configured ones. +#[tokio::test] +async fn apply_accepts_safe_imported_categories() { + let (state, _tempdir) = test_state(); + let preview = preview_with_categories(serde_json::json!([ + {"name": "tv", "output_dir": "shows/tv", "post_processing": 3}, + {"name": "movies", "output_dir": "/srv/media/movies", "post_processing": 3}, + ])); + + h_setup_apply(State(state.clone()), Json(preview)) + .await + .expect("safe categories should be accepted"); + + let names: Vec<_> = state + .config() + .categories + .iter() + .map(|c| c.name.clone()) + .collect(); + assert_eq!(names, ["tv", "movies"]); +} diff --git a/apps/rustnzb/tests/server_bind_error_test.rs b/apps/rustnzb/tests/server_bind_error_test.rs index 6d261f06..6595b46c 100644 --- a/apps/rustnzb/tests/server_bind_error_test.rs +++ b/apps/rustnzb/tests/server_bind_error_test.rs @@ -69,8 +69,6 @@ async fn serve_reports_context_on_bind_failure() { let state = build_state(config).await; let router = build_router(state.clone()); - #[cfg(feature = "webdav")] - let router = router.layer(axum::Extension(None::>)); let result = rustnzb::server::serve(state, router).await; let err = match result { @@ -103,8 +101,6 @@ async fn serve_succeeds_on_free_port() { let state = build_state(config).await; let router = build_router(state.clone()); - #[cfg(feature = "webdav")] - let router = router.layer(axum::Extension(None::>)); let handle = tokio::spawn(async move { rustnzb::server::serve(state, router).await }); diff --git a/apps/rustnzb/tests/support/mod.rs b/apps/rustnzb/tests/support/mod.rs index f3f8b6ce..86502bb5 100644 --- a/apps/rustnzb/tests/support/mod.rs +++ b/apps/rustnzb/tests/support/mod.rs @@ -26,6 +26,8 @@ fn workspace_path(path: &str) -> PathBuf { pub struct TestApp { pub base_url: String, pub complete_dir: PathBuf, + /// Shared app state behind the router, for seeding fixtures directly. + pub state: Arc, _tmp_dir: TempDir, handle: JoinHandle<()>, } @@ -123,9 +125,7 @@ async fn start_test_server_inner( credential_store, )); - let router = build_router(state); - #[cfg(feature = "webdav")] - let router = router.layer(axum::Extension(None::>)); + let router = build_router(state.clone()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("Failed to bind random port"); @@ -141,6 +141,7 @@ async fn start_test_server_inner( TestApp { base_url, complete_dir, + state, _tmp_dir: tmp_dir, handle, } diff --git a/crates/nzb-core/Cargo.toml b/crates/nzb-core/Cargo.toml index 41f52697..8a7e5016 100644 --- a/crates/nzb-core/Cargo.toml +++ b/crates/nzb-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nzb-core" -version = "0.2.18" +version = "0.2.19" edition = "2024" description = "Shared models, config, NZB parser, and SQLite database for NZB clients" license = "MIT" diff --git a/crates/nzb-core/src/config.rs b/crates/nzb-core/src/config.rs index d2dd1ab2..433c478e 100644 --- a/crates/nzb-core/src/config.rs +++ b/crates/nzb-core/src/config.rs @@ -57,7 +57,8 @@ pub struct GeneralConfig { /// History retention: how many NZBs to keep in history. /// `None` or `Some(0)` both mean keep all; see [`normalize_history_retention`]. pub history_retention: Option, - /// Max number of NZBs downloading simultaneously (default 1) + /// Max number of NZBs downloading simultaneously (default 1). + /// `0` means unlimited: every queued job may download at once. pub max_active_downloads: usize, /// Max number of independent jobs in post-processing at once. #[serde(default = "default_max_post_processing_jobs")] @@ -321,6 +322,30 @@ impl Default for CategoryConfig { } } +impl CategoryConfig { + /// Check the name and `output_dir` against the rules + /// `QueueManager::output_dir_for` applies when a job is enqueued, so an + /// entry that would make every job in the category fail can be refused + /// when it is saved. Returns a message suitable for an API client. + pub fn validate(&self) -> std::result::Result<(), &'static str> { + if crate::path::safe_component(&self.name).is_none() { + return Err("category name must be a single non-empty path component \ + without '/', '\\', '..' or control characters"); + } + if let Some(output_dir) = &self.output_dir { + // Only the shape of the override matters here, not where the + // complete directory lives. + if crate::path::category_output_root(std::path::Path::new("/"), output_dir).is_none() { + return Err( + "category output_dir must be an absolute path, or a relative \ + path below the complete directory, without '..' or control characters", + ); + } + } + Ok(()) + } +} + /// RSS feed configuration for automatic NZB downloading. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RssFeedConfig { @@ -328,7 +353,8 @@ pub struct RssFeedConfig { pub name: String, /// Feed URL (RSS 2.0 or Atom) pub url: String, - /// How often to poll, in seconds (default 900 = 15 minutes) + /// How often to poll, in seconds (default 900 = 15 minutes, minimum 60; + /// lower values are rejected by the API and clamped by the RSS monitor) #[serde(default = "default_poll_interval")] pub poll_interval_secs: u64, /// Category to assign to downloaded NZBs @@ -695,6 +721,31 @@ ssl = false assert!(cfg.server("nonexistent").is_none()); } + #[test] + fn category_validate_matches_enqueue_rules() { + let category = |name: &str, output_dir: Option<&str>| CategoryConfig { + name: name.into(), + output_dir: output_dir.map(PathBuf::from), + ..CategoryConfig::default() + }; + assert!(CategoryConfig::default().validate().is_ok()); + assert!(category("*", None).validate().is_ok()); + assert!(category("tv", Some("shows/tv")).validate().is_ok()); + assert!(category("tv", Some("/srv/media/tv")).validate().is_ok()); + for bad in [ + category("", None), + category("..", None), + category("a/b", None), + category("a\\b", None), + category("a\tb", None), + category("tv", Some("")), + category("tv", Some("../../../etc")), + category("tv", Some("/srv/../etc")), + ] { + assert!(bad.validate().is_err(), "{bad:?}"); + } + } + #[test] fn test_find_category_or_default() { let mut cfg = AppConfig::default(); diff --git a/crates/nzb-core/src/db.rs b/crates/nzb-core/src/db.rs index 70c1e76f..b6aa8883 100644 --- a/crates/nzb-core/src/db.rs +++ b/crates/nzb-core/src/db.rs @@ -405,6 +405,50 @@ impl Database { )?; } + if version < 14 { + info!("Applying database migration v14: history post-processing level"); + // Existing rows keep NULL: their level was never recorded. + let has_history: i64 = self.conn.query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'history'", + [], + |row| row.get(0), + )?; + if has_history > 0 { + self.conn + .execute_batch("ALTER TABLE history ADD COLUMN post_processing INTEGER;")?; + } + self.conn.execute_batch( + " + DELETE FROM schema_version; + INSERT INTO schema_version (version) VALUES (14); + ", + )?; + } + + if version < 15 { + info!("Applying database migration v15: per-job archive deletion"); + // Partial schemas (as built by migration tests) may lack either + // table; only add the column where the table exists. + for table in ["queue", "history"] { + let exists: i64 = self.conn.query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + )?; + if exists > 0 { + self.conn.execute_batch(&format!( + "ALTER TABLE {table} ADD COLUMN delete_archives INTEGER;" + ))?; + } + } + self.conn.execute_batch( + " + DELETE FROM schema_version; + INSERT INTO schema_version (version) VALUES (15); + ", + )?; + } + Ok(()) } @@ -470,9 +514,9 @@ impl Database { "INSERT INTO queue (id, name, category, status, priority, total_bytes, downloaded_bytes, file_count, files_completed, article_count, articles_downloaded, articles_failed, added_at, work_dir, output_dir, password, - nzb_raw, pp_override) + nzb_raw, pp_override, delete_archives) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, - ?16, ?17, ?18)", + ?16, ?17, ?18, ?19)", params![ job.id, job.name, @@ -492,6 +536,7 @@ impl Database { job.password, nzb_data, job.pp_override, + job.delete_archives, ], )?; @@ -600,9 +645,9 @@ impl Database { "INSERT INTO queue (id, name, category, status, priority, total_bytes, downloaded_bytes, file_count, files_completed, article_count, articles_downloaded, articles_failed, added_at, work_dir, output_dir, password, - pp_override) + pp_override, delete_archives) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, - ?17)", + ?17, ?18)", params![ job.id, job.name, @@ -621,6 +666,7 @@ impl Database { job.output_dir.to_string_lossy().to_string(), job.password, job.pp_override, + job.delete_archives, ], )?; Ok(()) @@ -685,7 +731,8 @@ impl Database { let mut stmt = self.conn.prepare( "SELECT id, name, category, status, priority, total_bytes, downloaded_bytes, file_count, files_completed, article_count, articles_downloaded, articles_failed, - added_at, completed_at, work_dir, output_dir, password, error_message, pp_override + added_at, completed_at, work_dir, output_dir, password, error_message, pp_override, + delete_archives FROM queue ORDER BY priority DESC, added_at ASC", )?; @@ -714,6 +761,7 @@ impl Database { error_message: row.get(17)?, speed_bps: 0, pp_override: row.get(18)?, + delete_archives: row.get(19)?, server_stats: Vec::new(), files: Vec::new(), // Loaded separately }) @@ -740,8 +788,9 @@ impl Database { self.conn.execute( "INSERT INTO history (id, name, category, status, total_bytes, downloaded_bytes, added_at, completed_at, download_time_secs, output_dir, stages, error_message, - nzb_data, server_stats, retry_data, failure_code) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)", + nzb_data, server_stats, retry_data, failure_code, post_processing, delete_archives) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, + ?17, ?18)", params![ entry.id, entry.name, @@ -759,6 +808,8 @@ impl Database { server_stats_json, entry.retry_data, entry.failure_code.map(|code| code.to_string()), + entry.post_processing, + entry.delete_archives, ], )?; @@ -823,7 +874,8 @@ impl Database { let mut stmt = self.conn.prepare( "SELECT id, name, category, status, total_bytes, downloaded_bytes, added_at, completed_at, download_time_secs, output_dir, stages, error_message, server_stats, - CASE WHEN nzb_data IS NOT NULL THEN 1 ELSE 0 END as has_nzb, failure_code + CASE WHEN nzb_data IS NOT NULL THEN 1 ELSE 0 END as has_nzb, failure_code, + post_processing, delete_archives FROM history ORDER BY completed_at DESC LIMIT ?1", )?; @@ -851,6 +903,8 @@ impl Database { stages, error_message: row.get(11)?, failure_code: parse_failure_code(row.get(14)?)?, + post_processing: row.get(15)?, + delete_archives: row.get(16)?, server_stats, // Don't load actual blob in list - just note if it exists nzb_data: if has_nzb != 0 { Some(Vec::new()) } else { None }, @@ -914,7 +968,7 @@ impl Database { let mut stmt = self.conn.prepare( "SELECT id, name, category, status, total_bytes, downloaded_bytes, added_at, completed_at, download_time_secs, output_dir, stages, error_message, server_stats, - failure_code + failure_code, post_processing, delete_archives FROM history WHERE id = ?1", )?; @@ -939,6 +993,8 @@ impl Database { stages, error_message: row.get(11)?, failure_code: parse_failure_code(row.get(13)?)?, + post_processing: row.get(14)?, + delete_archives: row.get(15)?, server_stats, nzb_data: None, retry_data: None, @@ -1367,6 +1423,7 @@ mod tests { error_message: None, speed_bps: 0, pp_override: None, + delete_archives: None, server_stats: Vec::new(), files: Vec::new(), } @@ -1387,6 +1444,38 @@ mod tests { assert_eq!(pp("plain-job"), None); } + #[test] + fn queue_and_history_round_trip_delete_archives() { + let db = Database::open_memory().unwrap(); + let mut keep = make_job("keep-job", "Keep Job"); + keep.delete_archives = Some(false); + db.queue_insert(&keep).unwrap(); + db.queue_insert(&make_job("plain-job", "Plain Job")) + .unwrap(); + let jobs = db.queue_list().unwrap(); + let flag = |id: &str| { + jobs.iter() + .find(|job| job.id == id) + .unwrap() + .delete_archives + }; + assert_eq!(flag("keep-job"), Some(false)); + assert_eq!(flag("plain-job"), None); + + let mut stored = make_history("keep-history", "Keep History"); + stored.delete_archives = Some(false); + db.history_insert(&stored).unwrap(); + assert_eq!( + db.history_get("keep-history") + .unwrap() + .unwrap() + .delete_archives, + Some(false) + ); + let listed = db.history_list(10).unwrap(); + assert_eq!(listed[0].delete_archives, Some(false)); + } + fn make_history(id: &str, name: &str) -> HistoryEntry { HistoryEntry { id: id.into(), @@ -1407,6 +1496,8 @@ mod tests { }], error_message: None, failure_code: None, + post_processing: None, + delete_archives: None, server_stats: Vec::new(), nzb_data: None, retry_data: None, @@ -1530,6 +1621,31 @@ mod tests { assert!(db.queue_list().unwrap().is_empty()); } + #[test] + fn history_round_trips_post_processing_level() { + let db = Database::open_memory().unwrap(); + let mut stored = make_history("pp-level", "PP Level"); + stored.post_processing = Some(1); + db.history_insert(&stored).unwrap(); + db.history_insert(&make_history("pp-legacy", "Legacy")) + .unwrap(); + + assert_eq!( + db.history_get("pp-level").unwrap().unwrap().post_processing, + Some(1) + ); + assert_eq!( + db.history_get("pp-legacy") + .unwrap() + .unwrap() + .post_processing, + None + ); + let listed = db.history_list(10).unwrap(); + let level = listed.iter().find(|entry| entry.id == "pp-level").unwrap(); + assert_eq!(level.post_processing, Some(1)); + } + #[test] fn typed_failure_migration_backfills_only_failed_history() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/nzb-core/src/models.rs b/crates/nzb-core/src/models.rs index c5661394..3938d122 100644 --- a/crates/nzb-core/src/models.rs +++ b/crates/nzb-core/src/models.rs @@ -190,6 +190,11 @@ pub struct NzbJob { /// `None` uses the category setting. #[serde(default)] pub pp_override: Option, + /// Whether a successful unpack deletes the archive volumes it extracted. + /// `Some(false)` keeps them (SABnzbd `pp=2`, "+Unpack" without + /// "+Delete"); `None` or `Some(true)` deletes them, the default. + #[serde(default)] + pub delete_archives: Option, /// Files in this job #[serde(skip)] pub files: Vec, @@ -249,6 +254,16 @@ pub struct HistoryEntry { pub stages: Vec, pub error_message: Option, pub failure_code: Option, + /// Effective post-processing level the job ran with (0=none, 1=repair, + /// 2=unpack, 3=repair+unpack). `None` for rows recorded before it was + /// stored. + #[serde(default)] + pub post_processing: Option, + /// The job's [`NzbJob::delete_archives`] flag, so history reports an + /// unpack that kept its archives. `None` for rows recorded before it was + /// stored, or when the default (delete) applied. + #[serde(default)] + pub delete_archives: Option, /// Per-server download statistics #[serde(default)] pub server_stats: Vec, @@ -374,7 +389,7 @@ pub struct RssRule { pub id: String, /// Human-readable name for the rule pub name: String, - /// Which feed(s) this rule applies to (one or more feed names) + /// Which feed(s) this rule applies to; empty means every feed pub feed_names: Vec, /// Category to assign to downloaded NZBs pub category: Option, diff --git a/crates/nzb-core/src/nzb_parser.rs b/crates/nzb-core/src/nzb_parser.rs index c8f49099..867a2bdd 100644 --- a/crates/nzb-core/src/nzb_parser.rs +++ b/crates/nzb-core/src/nzb_parser.rs @@ -284,6 +284,7 @@ pub fn parse_nzb(name: &str, data: &[u8]) -> Result { error_message: None, speed_bps: 0, pp_override: None, + delete_archives: None, server_stats: Vec::new(), files, }) @@ -338,6 +339,9 @@ fn sanitize_filename(name: &str) -> String { '<' | '>' | ':' | '"' | '|' | '?' | '*' => {} // Control characters (including null) c if c.is_control() => {} + // Bidi overrides/isolates/marks and zero-width spaces enable + // RTL-override filename spoofing; ZWJ/ZWNJ are kept. + c if crate::path::is_bidi_or_invisible_control(c) => {} // Backslash → forward slash is also risky; strip it '\\' => {} _ => out.push(ch), @@ -928,6 +932,31 @@ mod tests { assert_eq!(sanitize_filename(nfd), "caf\u{00E9}.rar"); } + #[test] + fn test_sanitize_strips_bidi_and_invisible_controls() { + assert_eq!(sanitize_filename("clip\u{202E}4pm.exe"), "clip4pm.exe"); + assert_eq!( + sanitize_filename("\u{FEFF}\u{2066}a\u{200B}b\u{200E}.mkv\u{2069}"), + "ab.mkv" + ); + assert_eq!(sanitize_filename("\u{202E}\u{200F}"), "unnamed"); + } + + #[test] + fn test_sanitize_keeps_joiners_cjk_and_accents() { + let family = "\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}.jpg"; + assert_eq!(sanitize_filename(family), family); + let persian = "\u{0645}\u{06CC}\u{200C}\u{062E}\u{0648}\u{0627}\u{0647}\u{0645}.txt"; + assert_eq!(sanitize_filename(persian), persian); + for text in [ + "進撃の巨人 第1話.mkv", + "Amélie.Poulain.mkv", + "Ñandú über straße.rar", + ] { + assert_eq!(sanitize_filename(text), text); + } + } + #[test] fn test_parse_nzb_normalizes_job_name() { let nzb_data = br#" diff --git a/crates/nzb-core/src/path.rs b/crates/nzb-core/src/path.rs index a30a7ea2..7b83e306 100644 --- a/crates/nzb-core/src/path.rs +++ b/crates/nzb-core/src/path.rs @@ -25,6 +25,33 @@ pub fn safe_join(root: &Path, name: &str) -> Option { output.starts_with(root).then_some(output) } +/// Whether `ch` is an invisible bidirectional or zero-width formatting +/// character that must not survive into a job or file name. +/// +/// This covers the bidi embeddings and overrides (U+202A..=U+202E), the bidi +/// isolates (U+2066..=U+2069), the directional marks LRM, RLM and ALM +/// (U+200E, U+200F, U+061C), ZERO WIDTH SPACE (U+200B) and the byte order +/// mark / ZERO WIDTH NO-BREAK SPACE (U+FEFF). An RTL override lets a name +/// such as `clip\u{202E}4pm.exe` display as `clipexe.mp4`. +/// +/// It deliberately does not cover every Unicode `Cf` character: ZERO WIDTH +/// JOINER (U+200D) and ZERO WIDTH NON-JOINER (U+200C) are required by emoji +/// sequences (family, flag and profession emoji) and by scripts such as +/// Persian and the Indic scripts, so stripping them would corrupt +/// legitimate names. +pub(crate) fn is_bidi_or_invisible_control(ch: char) -> bool { + matches!( + ch, + '\u{202A}'..='\u{202E}' + | '\u{2066}'..='\u{2069}' + | '\u{200E}' + | '\u{200F}' + | '\u{061C}' + | '\u{200B}' + | '\u{FEFF}' + ) +} + /// Validate a user supplied directory or category component. /// These values are intentionally a single path component. pub fn safe_component(value: &str) -> Option<&str> { @@ -34,12 +61,30 @@ pub fn safe_component(value: &str) -> Option<&str> { || value.contains('/') || value.contains('\\') || value.chars().any(char::is_control) + || value.chars().any(is_bidi_or_invisible_control) { return None; } Some(value) } +/// Resolve a category `output_dir` override to the root its jobs land in. +/// +/// A relative override is joined beneath `complete_dir` with [`safe_join`]. +/// An absolute override is used as is, provided it has no `..` component +/// and no control characters. Returns `None` for anything else, including +/// an empty override. +pub fn category_output_root(complete_dir: &Path, output_dir: &Path) -> Option { + if output_dir.is_absolute() { + let clean = !output_dir + .components() + .any(|component| matches!(component, std::path::Component::ParentDir)) + && !output_dir.to_string_lossy().chars().any(char::is_control); + return clean.then(|| output_dir.to_path_buf()); + } + safe_join(complete_dir, &output_dir.to_string_lossy()) +} + /// Maximum length, in bytes, of a sanitized job name. Leaves headroom under /// the common 255-byte `NAME_MAX` for suffixes such as `.1` or `_UNPACK_`. pub const MAX_JOB_NAME_BYTES: usize = 240; @@ -56,6 +101,9 @@ pub const UNNAMED_JOB: &str = "unnamed"; /// `Star Trek: Discovery` reads `Star Trek - Discovery`. /// - The other Windows-illegal characters `< > " / \ | ? *` and control /// characters become `_`. +/// - Bidi controls and zero-width spaces (U+202A-202E, U+2066-2069, +/// U+200E, U+200F, U+061C, U+200B, U+FEFF) are removed. ZWJ and ZWNJ +/// are kept because emoji sequences and some scripts need them. /// - Leading and trailing whitespace and dots are trimmed. /// - Windows reserved device names (`CON`, `PRN`, `AUX`, `NUL`, `COM1`-`COM9`, /// `LPT1`-`LPT9`, case-insensitive, with or without an extension) get a @@ -80,6 +128,7 @@ pub fn sanitize_job_name(name: &str) -> String { } '<' | '>' | '"' | '/' | '\\' | '|' | '?' | '*' => replaced.push('_'), c if c.is_control() => replaced.push('_'), + c if is_bidi_or_invisible_control(c) => {} c => replaced.push(c), } } @@ -108,6 +157,109 @@ pub fn sanitize_job_name(name: &str) -> String { out } +/// Split SABnzbd's inline job-password conventions off a job name. +/// +/// A port of SABnzbd 5.0.4 `misc.scan_password`. Recognised forms, in +/// SABnzbd's order of precedence: +/// +/// - `name/password` and `name / password`, when the `/` precedes any `{{` +/// - `name password=password` +/// - `name{{password}}` (the last `}}` closes it, so the password is greedy) +/// +/// A trailing `.nzb`, `.par` or `.par2` extension is ignored. Returns +/// `Some((name, password))` when a password was found, `None` otherwise +/// (including for `http://` and `https://` URLs). The password may be empty, +/// as in `name{{}}`; the name never is. +/// +/// Callers apply this to the raw client-supplied name *before* +/// [`sanitize_job_name`], which would otherwise turn `/` into `_` and keep +/// the braces in the folder name. +pub fn split_job_password(name: &str) -> Option<(String, String)> { + if name.contains("http://") || name.contains("https://") { + return None; + } + let name = strip_nzb_extensions(name); + let strip = |s: &str| s.trim_matches(['.', ' ']).to_string(); + + // `{{` counts only after the first character, as in SABnzbd. + let first_len = name.chars().next().map_or(0, char::len_utf8); + let braces = name[first_len..] + .find("{{") + .map_or(name.len(), |index| index + first_len); + let slash = name.find('/'); + + if let Some(slash) = slash + && 0 < slash + && slash < braces + && !name.contains("password=") + { + // `name / password`: drop the spaces around the slash. + if name.find(" / ").map(|index| index + 1) == Some(slash) { + let head = strip(&name[..slash - 1]); + if !head.is_empty() { + return Some((head, name[slash + 2..].to_string())); + } + } + let head = strip(&name[..slash]); + if !head.is_empty() { + return Some((head, name[slash + 1..].to_string())); + } + } + + if let Some(pw) = name.find("password=") + && pw > 0 + { + let head = strip(&name[..pw]); + if !head.is_empty() { + return Some((head, name[pw + "password=".len()..].to_string())); + } + } + + if braces < name.len() + && let Some(closing) = name.rfind("}}") + && closing > braces + { + let head = strip(&name[..braces]); + if !head.is_empty() { + return Some((head, name[braces + 2..closing].to_string())); + } + } + + if let Some(slash) = slash + && slash > 0 + { + let head = strip(&name[..slash]); + if !head.is_empty() { + return Some((head, name[slash + 1..].to_string())); + } + } + + None +} + +/// Strip trailing `.nzb`/`.par`/`.par2` extensions (case-insensitive), as +/// SABnzbd's `strip_extensions` does via `os.path.splitext`. +fn strip_nzb_extensions(mut name: &str) -> &str { + loop { + let base_start = name.rfind('/').map_or(0, |index| index + 1); + let base = &name[base_start..]; + // splitext ignores leading dots of the final component. + let leading = base.len() - base.trim_start_matches('.').len(); + let Some(dot) = base[leading..].rfind('.').map(|index| index + leading) else { + return name; + }; + let ext = &base[dot..]; + if [".nzb", ".par", ".par2"] + .iter() + .any(|known| ext.eq_ignore_ascii_case(known)) + { + name = &name[..base_start + dot]; + } else { + return name; + } + } +} + /// Whether `name` is a Windows reserved device name. Windows ignores the /// extension and trailing spaces, so `con.txt` and `NUL .x` count too. fn is_windows_reserved_name(name: &str) -> bool { @@ -140,6 +292,31 @@ mod tests { assert!(safe_component(r"movies\tv").is_none()); } + #[test] + fn category_output_root_accepts_subdirs_and_clean_absolute_roots() { + let complete = Path::new("/data/complete"); + assert_eq!( + category_output_root(complete, Path::new("shows/tv")), + Some(PathBuf::from("/data/complete/shows/tv")) + ); + assert_eq!( + category_output_root(complete, Path::new("/srv/media")), + Some(PathBuf::from("/srv/media")) + ); + for unsafe_dir in [ + "", + "../../../etc", + "shows/../../etc", + "/srv/../etc", + "/srv/a\nb", + ] { + assert!( + category_output_root(complete, Path::new(unsafe_dir)).is_none(), + "{unsafe_dir:?}" + ); + } + } + #[test] fn sanitize_job_name_replaces_windows_illegal_characters() { assert_eq!(sanitize_job_name("a:b*c?d"), "a-b_c_d"); @@ -234,4 +411,117 @@ mod tests { // NFD input is normalized to NFC. assert_eq!(sanitize_job_name("cafe\u{0301}"), "caf\u{00E9}"); } + + /// SABnzbd 5.0.4 `tests/test_misc.py::test_scan_password`. + #[test] + fn split_job_password_matches_sabnzbd_scan_password() { + let cases: &[(&str, &str, Option<&str>)] = &[ + ( + "my_awesome_nzb_file{{password}}", + "my_awesome_nzb_file", + Some("password"), + ), + ( + "file_with_text_after_pw{{passw0rd}}_[180519]", + "file_with_text_after_pw", + Some("passw0rd"), + ), + ("file_without_pw", "file_without_pw", None), + ( + "multiple_pw{{first-pw}}_{{second-pw}}", + "multiple_pw", + Some("first-pw}}_{{second-pw"), + ), + ("デビアン", "デビアン", None), + ( + "Gentoo_Hobby_Edition {{secret}}", + "Gentoo_Hobby_Edition", + Some("secret"), + ), + ("Test {{secret}}.nzb", "Test", Some("secret")), + ("Mandrake{{top{{secret}}", "Mandrake", Some("top{{secret")), + ("Красная}}{{Шляпа}}", "Красная}}", Some("Шляпа")), + ("{{Jobname{{PassWord}}", "{{Jobname", Some("PassWord")), + ("Hello/kITTY", "Hello", Some("kITTY")), + ("Hello/kITTY.nzb", "Hello", Some("kITTY")), + ("/Jobname", "/Jobname", None), + ("Jobname/Top{{Secret}}", "Jobname", Some("Top{{Secret}}")), + ("Jobname / Top{{Secret}}", "Jobname", Some("Top{{Secret}}")), + ( + "Jobname / Top{{Secret}}.nzb", + "Jobname", + Some("Top{{Secret}}"), + ), + ("לינוקס/معلومات سرية", "לינוקס", Some("معلومات سرية")), + ("לינוקס{{معلومات سرية}}", "לינוקס", Some("معلومات سرية")), + ( + "thư điện tử password=mật_khẩu", + "thư điện tử", + Some("mật_khẩu"), + ), + ( + "password=PartOfTheJobname", + "password=PartOfTheJobname", + None, + ), + ("Job password=Test.par2", "Job", Some("Test")), + ("Job}}Name{{FTW", "Job}}Name{{FTW", None), + ("./Text", "./Text", None), + ]; + for (input, name, password) in cases { + match (split_job_password(input), password) { + (Some((got_name, got_pw)), Some(pw)) => { + assert_eq!( + (got_name.as_str(), got_pw.as_str()), + (*name, *pw), + "{input}" + ); + } + (None, None) => {} + (got, _) => panic!("{input}: got {got:?}, expected {name:?}/{password:?}"), + } + } + assert_eq!(split_job_password("https://host/a{{b}}.nzb"), None); + assert_eq!( + split_job_password("Show{{}}"), + Some(("Show".to_string(), String::new())) + ); + } + + #[test] + fn sanitize_job_name_strips_bidi_and_invisible_controls() { + // RTL override spoofing: "clip\u{202E}4pm.exe" displays as "clipexe.mp4". + assert_eq!(sanitize_job_name("clip\u{202E}4pm.exe"), "clip4pm.exe"); + for ch in [ + '\u{202A}', '\u{202B}', '\u{202C}', '\u{202D}', '\u{202E}', '\u{2066}', '\u{2067}', + '\u{2068}', '\u{2069}', '\u{200E}', '\u{200F}', '\u{061C}', '\u{200B}', '\u{FEFF}', + ] { + let input = format!("a{ch}b"); + assert_eq!(sanitize_job_name(&input), "ab", "{:04X}", ch as u32); + } + // A name made only of invisible characters falls back to the default. + assert_eq!(sanitize_job_name("\u{202E}\u{FEFF}"), UNNAMED_JOB); + } + + #[test] + fn sanitize_job_name_keeps_joiners_cjk_and_accents() { + // ZWJ emoji sequence (family) and ZWNJ (Persian) must survive. + let family = "\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}"; + assert_eq!(sanitize_job_name(family), family); + let persian = "\u{0645}\u{06CC}\u{200C}\u{062E}\u{0648}\u{0627}\u{0647}\u{0645}"; + assert_eq!(sanitize_job_name(persian), persian); + for text in ["進撃の巨人 第1話", "Amélie Poulain", "Ñandú über straße"] { + assert_eq!(sanitize_job_name(text), text); + } + } + + #[test] + fn safe_component_rejects_bidi_controls() { + assert!(safe_component("movies\u{202E}vka").is_none()); + assert!(safe_component("\u{2066}tv\u{2069}").is_none()); + assert!(safe_component("tv\u{200F}").is_none()); + assert!(safe_component("tv\u{FEFF}").is_none()); + assert!(safe_component("\u{1F468}\u{200D}\u{1F469}").is_some()); + assert!(safe_component("映画").is_some()); + } } diff --git a/crates/nzb-core/src/sabnzbd_import.rs b/crates/nzb-core/src/sabnzbd_import.rs index 5b3535d0..0f430881 100644 --- a/crates/nzb-core/src/sabnzbd_import.rs +++ b/crates/nzb-core/src/sabnzbd_import.rs @@ -216,15 +216,29 @@ pub fn parse_sabnzbd_ini(content: &str) -> SabnzbdImportPreview { let servers: Vec = sections .iter() .filter(|((section, subsection), _)| section == "servers" && !subsection.is_empty()) - .map(|((_, _), kv)| build_imported_server(kv, false)) + .map(|((_, subsection), kv)| { + let mut server = build_imported_server(kv, false); + // SABnzbd keys servers by their `[[section]]` header; fall back to + // it when the INI carries no (or an empty) inner name. + if server.name.is_empty() { + server.name = subsection.clone(); + } + server + }) .collect(); // --- Categories (from [categories] → [[name]]) --- let categories: Vec = sections .iter() .filter(|((section, subsection), _)| section == "categories" && !subsection.is_empty()) - .map(|((_, _), kv)| { - let name = kv.get("name").map(|s| s.as_str()).unwrap_or("*"); + .map(|((_, subsection), kv)| { + // The `[[section]]` header is SABnzbd's key for the category; + // use it when the inner `name` is absent or empty. + let name = kv + .get("name") + .map(|s| s.as_str()) + .filter(|s| !s.is_empty()) + .unwrap_or(subsection); let name = if name == "*" { "Default" } else { name }; // Check for scripts @@ -461,6 +475,7 @@ fn build_imported_server(kv: &HashMap, from_api: bool) -> Import ImportedServer { name: kv .get("displayname") + .filter(|s| !s.is_empty()) .or(kv.get("name")) .cloned() .unwrap_or_default(), @@ -555,6 +570,53 @@ mod tests { ); } + /// The `[[section]]` header is the key SABnzbd stores categories and + /// servers under; an INI without an inner `name =` must fall back to it + /// instead of importing every category as "Default" and nameless servers. + #[test] + fn ini_import_falls_back_to_section_header_for_names() { + let preview = parse_sabnzbd_ini( + r#" + [servers] + [[news.example.test]] + host = news.example.test + [[backup]] + name = "" + displayname = "" + host = backup.example.test + [categories] + [[*]] + pp = 3 + [[tv]] + dir = /complete/tv + [[movies]] + name = "" + "#, + ); + + let mut servers: Vec<_> = preview + .servers + .iter() + .map(|server| (server.name.as_str(), server.host.as_str())) + .collect(); + servers.sort(); + assert_eq!( + servers, + vec![ + ("backup", "backup.example.test"), + ("news.example.test", "news.example.test"), + ] + ); + + let mut categories: Vec<_> = preview + .categories + .iter() + .map(|category| category.name.as_str()) + .collect(); + categories.sort(); + assert_eq!(categories, vec!["Default", "movies", "tv"]); + } + #[test] fn api_import_handles_string_numbers_and_masked_passwords() { let preview = parse_sabnzbd_api_response(&serde_json::json!({ diff --git a/crates/nzb-dispatch/src/download_engine.rs b/crates/nzb-dispatch/src/download_engine.rs index fdb2cdf8..fb78a0e2 100644 --- a/crates/nzb-dispatch/src/download_engine.rs +++ b/crates/nzb-dispatch/src/download_engine.rs @@ -3434,6 +3434,7 @@ mod tests { error_message: None, speed_bps: 0, pp_override: None, + delete_archives: None, server_stats: Vec::new(), files: Vec::new(), } diff --git a/crates/nzb-postproc/Cargo.toml b/crates/nzb-postproc/Cargo.toml index c42a60e9..49ff6b7d 100644 --- a/crates/nzb-postproc/Cargo.toml +++ b/crates/nzb-postproc/Cargo.toml @@ -13,7 +13,7 @@ groups-db = ["nzb-core/groups-db"] [dependencies] nzb-core = { version = "0.2.18", path = "../nzb-core" } -tokio = { version = "1", features = ["rt", "sync", "process"] } +tokio = { version = "1", features = ["rt", "sync", "process", "io-util"] } tracing = "0.1" opentelemetry.workspace = true anyhow = "1" @@ -30,7 +30,7 @@ tempfile = "3" # packet MD5s. Version tracks what `rust-par2` itself parses with. md-5 = "0.11" crc32fast = "1" -tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } [lints.clippy] all = { level = "warn", priority = -1 } diff --git a/crates/nzb-postproc/src/lib.rs b/crates/nzb-postproc/src/lib.rs index daba0790..73ff52ed 100644 --- a/crates/nzb-postproc/src/lib.rs +++ b/crates/nzb-postproc/src/lib.rs @@ -23,7 +23,9 @@ pub use detect::{ pub use par2::recovery_can_cover; pub use pipeline::{ PostProcConfig, PostProcResult, run_pipeline, run_pipeline_with_cleanup, - run_pipeline_with_resources, + run_pipeline_with_resources, run_repair_pipeline, }; pub use resources::{PostProcLimits, PostProcResourcePool, PostProcResourceSnapshot}; -pub use unpack::{extract_tar, find_unrar, normalize_extracted_permissions}; +pub use unpack::{ + extract_tar, extractor_reads_password_from_stdin, find_unrar, normalize_extracted_permissions, +}; diff --git a/crates/nzb-postproc/src/pipeline.rs b/crates/nzb-postproc/src/pipeline.rs index 225bc21e..d539f999 100644 --- a/crates/nzb-postproc/src/pipeline.rs +++ b/crates/nzb-postproc/src/pipeline.rs @@ -123,6 +123,11 @@ pub struct PostProcResult { pub struct PostProcConfig { /// Remove par2 and archive files after successful extraction. pub cleanup_after_extract: bool, + /// Whether cleanup removes the archive volumes it extracted. When false + /// (SABnzbd `pp=2`, "+Unpack" without "+Delete") the archives are kept + /// and cleanup only removes PAR2 files and the category's cleanup + /// patterns / unwanted extensions. + pub delete_archives: bool, /// Directory where extracted files should be placed. /// If None, extracts into the job directory itself. pub output_dir: Option, @@ -148,6 +153,7 @@ impl Default for PostProcConfig { fn default() -> Self { Self { cleanup_after_extract: true, + delete_archives: true, output_dir: None, articles_failed: 0, content_articles_failed: 0, @@ -189,6 +195,38 @@ pub async fn run_pipeline_with_cleanup( resources: Option<&Arc>, cleanup_patterns: &[String], unwanted_extensions: &[String], +) -> PostProcResult { + run_stages( + job_dir, + config, + resources, + cleanup_patterns, + unwanted_extensions, + true, + ) + .await +} + +/// Run only PAR2 verification and repair, without extraction or cleanup. +/// +/// This is SABnzbd's "repair only" post-processing level (`pp=1`): the +/// archives themselves are the job's output, so they are neither unpacked +/// nor removed. +pub async fn run_repair_pipeline( + job_dir: &Path, + config: &PostProcConfig, + resources: Option<&Arc>, +) -> PostProcResult { + run_stages(job_dir, config, resources, &[], &[], false).await +} + +async fn run_stages( + job_dir: &Path, + config: &PostProcConfig, + resources: Option<&Arc>, + cleanup_patterns: &[String], + unwanted_extensions: &[String], + unpack: bool, ) -> PostProcResult { let mut stages: Vec = Vec::new(); let mut pipeline_ok = true; @@ -451,7 +489,7 @@ pub async fn run_pipeline_with_cleanup( // Extraction is safe only after verification/repair succeeded (or source // content was already known-good). Confirmed unrepaired content damage // must not be mistaken for a successful archive. - let should_extract = pipeline_ok; + let should_extract = unpack && pipeline_ok; let mut extracted_archives = Vec::new(); if should_extract { let output_dir = config.output_dir.as_deref().unwrap_or(job_dir); @@ -502,7 +540,7 @@ pub async fn run_pipeline_with_cleanup( // ------------------------------------------------------------------ // Stage 4: Cleanup // ------------------------------------------------------------------ - if pipeline_ok && config.cleanup_after_extract { + if unpack && pipeline_ok && config.cleanup_after_extract { let cleanup_root = config.output_dir.as_deref().unwrap_or(job_dir); let result = run_cleanup_stage_with_rules( job_dir, @@ -510,6 +548,7 @@ pub async fn run_pipeline_with_cleanup( &extracted_archives, cleanup_patterns, unwanted_extensions, + config.delete_archives, ); stages.push(result); } @@ -812,7 +851,7 @@ async fn run_extract_stage( #[cfg(test)] fn run_cleanup_stage(job_dir: &Path, extracted_archives: &[PathBuf]) -> StageResult { - run_cleanup_stage_with_rules(job_dir, job_dir, extracted_archives, &[], &[]) + run_cleanup_stage_with_rules(job_dir, job_dir, extracted_archives, &[], &[], true) } fn wildcard_match(pattern: &str, value: &str) -> bool { @@ -852,19 +891,29 @@ fn run_cleanup_stage_with_rules( extracted_archives: &[PathBuf], cleanup_patterns: &[String], unwanted_extensions: &[String], + delete_archives: bool, ) -> StageResult { let start = Instant::now(); let mut files = find_cleanup_files(job_dir); - files.extend( - extracted_archives - .iter() - .filter(|path| { - std::fs::symlink_metadata(path) - .map(|metadata| metadata.file_type().is_file()) - .unwrap_or(false) - }) - .cloned(), - ); + if delete_archives { + files.extend( + extracted_archives + .iter() + .filter(|path| { + std::fs::symlink_metadata(path) + .map(|metadata| metadata.file_type().is_file()) + .unwrap_or(false) + }) + .cloned(), + ); + } else { + // Keep the archive set; PAR2 recovery files are still removed, as + // SABnzbd does after a successful verify regardless of "+Delete". + files.retain(|path| { + path.extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("par2")) + }); + } let normalized_extensions: Vec = unwanted_extensions .iter() @@ -1117,6 +1166,32 @@ mod tests { assert!(!output.path().join("payload.txt").exists()); } + #[tokio::test] + async fn repair_pipeline_neither_extracts_nor_cleans_up() { + let source = tempfile::tempdir().unwrap(); + let output = tempfile::tempdir().unwrap(); + let archive = source.path().join("release.zip"); + write_zip(&archive, &[("payload.txt", b"payload")]); + + let config = PostProcConfig { + output_dir: Some(output.path().to_path_buf()), + ..Default::default() + }; + let result = run_repair_pipeline(source.path(), &config, None).await; + + assert!(result.success, "{result:?}"); + assert!( + result + .stages + .iter() + .all(|stage| stage.name != "Extract" && stage.name != "Cleanup"), + "{:?}", + result.stages + ); + assert!(archive.exists()); + assert!(!output.path().join("payload.txt").exists()); + } + #[tokio::test] async fn recursive_cleanup_removes_nested_archives_from_output_directory() { let source = tempfile::tempdir().unwrap(); @@ -1232,6 +1307,61 @@ mod tests { assert!(!subs.exists()); } + #[tokio::test] + async fn keep_archives_preserves_unpacked_rar_set_and_removes_par2() { + // SABnzbd pp=2 ("+Unpack" without "+Delete"): the RAR set is unpacked + // (here by direct unpack) but its volumes must survive cleanup. + let job_dir = tempfile::tempdir().unwrap(); + let output = tempfile::tempdir().unwrap(); + let volumes = ["movie.part01.rar", "movie.part02.rar", "movie.r00"]; + for volume in volumes { + fs::write(job_dir.path().join(volume), b"Rar!\x1a\x07\x01\x00 volume").unwrap(); + } + fs::write(job_dir.path().join("movie.par2"), b"par2").unwrap(); + fs::write(output.path().join("movie.mkv"), b"movie").unwrap(); + + let config = PostProcConfig { + output_dir: Some(output.path().to_path_buf()), + skip_extract: true, + delete_archives: false, + ..Default::default() + }; + let result = run_pipeline(job_dir.path(), &config).await; + + assert!(result.success, "{result:?}"); + for volume in volumes { + assert!(job_dir.path().join(volume).exists(), "{volume} was deleted"); + } + assert!(!job_dir.path().join("movie.par2").exists()); + assert_eq!(fs::read(output.path().join("movie.mkv")).unwrap(), b"movie"); + } + + #[tokio::test] + async fn keep_archives_preserves_extracted_zip_and_nested_archives() { + let source = tempfile::tempdir().unwrap(); + let output = tempfile::tempdir().unwrap(); + let inner = source.path().join("inner.zip"); + write_zip(&inner, &[("payload.txt", b"nested payload")]); + let outer = source.path().join("outer.zip"); + write_zip(&outer, &[("inner.zip", &fs::read(&inner).unwrap())]); + fs::remove_file(inner).unwrap(); + + let config = PostProcConfig { + output_dir: Some(output.path().to_path_buf()), + delete_archives: false, + ..Default::default() + }; + let result = run_pipeline(source.path(), &config).await; + + assert!(result.success, "{result:?}"); + assert_eq!( + fs::read(output.path().join("payload.txt")).unwrap(), + b"nested payload" + ); + assert!(outer.exists()); + assert!(output.path().join("inner.zip").exists()); + } + #[tokio::test] async fn direct_unpack_keeps_non_rar_archive_that_fails_to_extract() { let job_dir = tempfile::tempdir().unwrap(); diff --git a/crates/nzb-postproc/src/unpack.rs b/crates/nzb-postproc/src/unpack.rs index 5c7e791d..ee0bac38 100644 --- a/crates/nzb-postproc/src/unpack.rs +++ b/crates/nzb-postproc/src/unpack.rs @@ -27,21 +27,172 @@ pub struct UnpackResult { pub error_output: String, } -fn unrar_password_flag(password: Option<&str>) -> String { +/// How an archive password reaches the extractor. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum PasswordArg { + /// No password is known. + None, + /// Written to the extractor's stdin (one line, then EOF), so it never + /// appears in `ps` or `/proc//cmdline`. + Stdin, + /// Passed as `-p`, visible to other local users through `ps` + /// and `/proc//cmdline`. Used only for extractors that are not known + /// to read a piped password (see `docs/KNOWN_ISSUES.md`). + Argv(String), +} + +impl PasswordArg { + /// The password to write to stdin, if any. + fn stdin_password<'a>(&self, password: Option<&'a str>) -> Option<&'a str> { + match self { + Self::Stdin => password, + _ => None, + } + } + + /// How many password prompts the caller answers itself. + fn answered_prompts(&self) -> usize { + usize::from(*self == Self::Stdin) + } +} + +/// Choose how to hand `password` to the extractor `bin`. +async fn password_arg_for(bin: &str, password: Option<&str>) -> PasswordArg { + let Some(password) = password.filter(|pw| !pw.is_empty()) else { + return PasswordArg::None; + }; + // A password is read as one line, so one containing a line break can only + // be passed as an argument. + if !password.contains(['\n', '\r']) { + let probe_bin = bin.to_string(); + if tokio::task::spawn_blocking(move || extractor_reads_password_from_stdin(&probe_bin)) + .await + .unwrap_or(false) + { + return PasswordArg::Stdin; + } + } + PasswordArg::Argv(password.to_string()) +} + +/// Whether the extractor `bin` reads a password from a piped stdin when it +/// prompts for one (unrar given `-p` with no value; 7-Zip on an encrypted +/// archive), instead of from the terminal. +/// +/// Decided from the banner it prints when run with no arguments, and cached +/// per binary. Verified behaviour: rarlab UNRAR 6.21 and 7.20 and 7-Zip 25.01 +/// read the piped line even with a controlling terminal. Older unrar +/// releases, unrar-free and p7zip 16.02 (which uses `getpass`, i.e. the +/// terminal) are treated as unsupported and get the password as an argument. +pub fn extractor_reads_password_from_stdin(bin: &str) -> bool { + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + + static CACHE: OnceLock>> = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new())); + if let Some(&known) = cache.lock().unwrap_or_else(|e| e.into_inner()).get(bin) { + return known; + } + let supported = std::process::Command::new(bin) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .map(|output| { + let mut banner = String::from_utf8_lossy(&output.stdout).into_owned(); + banner.push('\n'); + banner.push_str(&String::from_utf8_lossy(&output.stderr)); + banner_reads_password_from_stdin(&banner) + }) + .unwrap_or(false); + cache + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(bin.to_string(), supported); + supported +} + +/// Parse an extractor banner: rarlab `UNRAR`/`RAR` 6 or later, or 7-Zip 21 +/// or later (the first 7-Zip release for Linux; not p7zip). +fn banner_reads_password_from_stdin(banner: &str) -> bool { + if banner.contains("p7zip") { + return false; + } + for line in banner.lines() { + let line = line.trim(); + let (rest, min_major) = if let Some(rest) = line + .strip_prefix("UNRAR ") + .or_else(|| line.strip_prefix("RAR ")) + { + (rest, 6) + } else if let Some(rest) = line.strip_prefix("7-Zip") { + (rest, 21) + } else { + continue; + }; + let major = rest.split_whitespace().find_map(|token| { + token + .split_once('.') + .and_then(|(major, _)| major.parse::().ok()) + }); + if let Some(major) = major { + return major >= min_major; + } + } + false +} + +/// Run an extractor, writing `stdin_password` (if any) as a single line to +/// its stdin and then closing it. Closing matters: a wrong password makes the +/// extractor fail or hit EOF on a re-prompt instead of waiting for input. +async fn run_extractor( + bin: &str, + args: Vec, + stdin_password: Option<&str>, +) -> std::io::Result { + let mut command = Command::new(bin); + command + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let Some(password) = stdin_password else { + return command.stdin(Stdio::null()).output().await; + }; + let mut child = command.stdin(Stdio::piped()).kill_on_drop(true).spawn()?; + if let Some(mut stdin) = child.stdin.take() { + use tokio::io::AsyncWriteExt; + // The line is far below the pipe buffer size, so this cannot block on + // an extractor that is busy writing output. An extractor that never + // asks (an unencrypted 7z) may already have exited: ignore EPIPE. + let _ = stdin.write_all(format!("{password}\n").as_bytes()).await; + let _ = stdin.shutdown().await; + } + child.wait_with_output().await +} + +/// The `-p` switch for unrar: `-p-` (never prompt) without a password, bare +/// `-p` (prompt, answered on stdin) or `-p`. +fn unrar_password_flag(password: &PasswordArg) -> String { match password { - Some(pw) => format!("-p{pw}"), - None => "-p-".to_string(), + PasswordArg::None => "-p-".to_string(), + PasswordArg::Stdin => "-p".to_string(), + PasswordArg::Argv(pw) => format!("-p{pw}"), } } -fn sevenz_password_arg(password: Option<&str>) -> Option { - password.map(|pw| format!("-p{pw}")) +/// The `-p` switch for 7z. Without a password and in stdin mode there is +/// none: 7z prompts by itself when the archive turns out to be encrypted. +fn sevenz_password_arg(password: &PasswordArg) -> Option { + match password { + PasswordArg::Argv(pw) => Some(format!("-p{pw}")), + PasswordArg::None | PasswordArg::Stdin => None, + } } fn rar_extract_args_with_7z( rar_file: &Path, output_dir: &Path, - password: Option<&str>, + password: &PasswordArg, ) -> Vec { let mut args = vec![ "x".to_string(), @@ -58,7 +209,7 @@ fn rar_extract_args_with_7z( fn rar_extract_args_with_unrar( rar_file: &Path, output_dir: &Path, - password: Option<&str>, + password: &PasswordArg, ) -> Vec { vec![ "x".to_string(), @@ -75,7 +226,7 @@ fn rar_extract_args_with_unrar( fn sevenz_extract_args( archive_file: &Path, output_dir: &Path, - password: Option<&str>, + password: &PasswordArg, ) -> Vec { let mut args = vec![ "x".to_string(), @@ -89,6 +240,30 @@ fn sevenz_extract_args( args } +/// Whether the extractor asked for a password more often than we answered: +/// with `-p` on stdin, unrar and 7z always print one prompt we answer. +fn unanswered_password_prompt(output: &str, password: &PasswordArg) -> bool { + output.matches("Enter password").count() > password.answered_prompts() +} + +/// Whether a failed unrar run was a missing or wrong password. unrar 6/7 +/// print `Incorrect password for ` and exit with code 11 (RARX_BADPWD). +fn unrar_password_failure(exit_code: Option, output: &str, password: &PasswordArg) -> bool { + exit_code == Some(11) + || output.contains("Incorrect password") + || output.contains("password is incorrect") + || output.contains("Encrypted file") + || unanswered_password_prompt(output, password) +} + +/// Whether a failed 7z run was a missing or wrong password. +fn sevenz_password_failure(output: &str, password: &PasswordArg) -> bool { + SEVENZ_PASSWORD_PATTERNS + .iter() + .any(|pattern| output.contains(pattern)) + || unanswered_password_prompt(output, password) +} + /// Return the regular files currently present below an extraction directory. /// Extractors do not expose a portable machine-readable file list, so a /// before/after snapshot is more reliable than parsing localized console text. @@ -257,8 +432,10 @@ fn read_process_umask() -> u32 { /// Extract RAR archives in a directory. /// -/// If `password` is `Some`, it is passed to the extractor (`-p` for unrar, -/// `-p` for 7z). When `None`, `-p-` is used to suppress password prompts. +/// If `password` is `Some`, it is written to the extractor's stdin when the +/// extractor is known to read it from there (unrar 6+, 7-Zip 21+), so it is +/// not visible in the process list; older extractors get `-p`. When +/// `None`, unrar gets `-p-` to suppress password prompts. pub async fn extract_rar( rar_file: &Path, output_dir: &Path, @@ -290,26 +467,16 @@ async fn extract_rar_with( std::fs::create_dir_all(output_dir)?; let before = output_files(output_dir)?; - let output = if use_7z { + let password_arg = password_arg_for(bin, password).await; + let args = if use_7z { // Do not pass `-p-` to 7z when no password is set. p7zip's built-in // RAR handler treats it like a passworded archive hint and fails on // valid multi-volume RAR sets. - Command::new(bin) - .args(rar_extract_args_with_7z(rar_file, output_dir, password)) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await? + rar_extract_args_with_7z(rar_file, output_dir, &password_arg) } else { - Command::new(bin) - .args(rar_extract_args_with_unrar(rar_file, output_dir, password)) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await? + rar_extract_args_with_unrar(rar_file, output_dir, &password_arg) }; + let output = run_extractor(bin, args, password_arg.stdin_password(password)).await?; let stdout = String::from_utf8_lossy(&output.stdout).to_string(); let stderr = String::from_utf8_lossy(&output.stderr).to_string(); @@ -322,10 +489,11 @@ async fn extract_rar_with( } if !success { - // Detect password-protected archives (unrar exit code 255 + password prompt) - let is_encrypted = combined.contains("Enter password") - || combined.contains("password is incorrect") - || combined.contains("Encrypted file"); + let is_encrypted = if use_7z { + sevenz_password_failure(&combined, &password_arg) + } else { + unrar_password_failure(output.status.code(), &combined, &password_arg) + }; if is_encrypted { warn!( file = %rar_file.display(), @@ -353,16 +521,19 @@ async fn extract_rar_with( }) } -/// Strings in 7z stderr/stdout that indicate a password-protected archive. +/// Strings in 7z stderr/stdout that indicate a missing or wrong password. +/// The `Enter password` prompt is handled separately: in stdin mode 7z +/// prints it once for every encrypted archive, whatever the outcome. const SEVENZ_PASSWORD_PATTERNS: &[&str] = &[ "Wrong password", "Can not open encrypted archive", - "Enter password", + "Cannot open encrypted archive", "ERROR: Data Error in encrypted file", "password is incorrect", ]; -/// Extract 7z archives by shelling out to the 7z binary. +/// Extract 7z archives by shelling out to the 7z binary. A password is fed +/// on stdin where the binary supports it, as for [`extract_rar`]. pub async fn extract_7z( archive_file: &Path, output_dir: &Path, @@ -370,19 +541,27 @@ pub async fn extract_7z( ) -> anyhow::Result { let sevenz_bin = find_7z().ok_or_else(|| anyhow::anyhow!("7z/7zz/7za binary not found on PATH"))?; + extract_7z_with(&sevenz_bin, archive_file, output_dir, password).await +} +async fn extract_7z_with( + sevenz_bin: &str, + archive_file: &Path, + output_dir: &Path, + password: Option<&str>, +) -> anyhow::Result { info!(file = %archive_file.display(), dest = %output_dir.display(), "Extracting 7z"); std::fs::create_dir_all(output_dir)?; let before = output_files(output_dir)?; - let output = Command::new(&sevenz_bin) - .args(sevenz_extract_args(archive_file, output_dir, password)) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await?; + let password_arg = password_arg_for(sevenz_bin, password).await; + let output = run_extractor( + sevenz_bin, + sevenz_extract_args(archive_file, output_dir, &password_arg), + password_arg.stdin_password(password), + ) + .await?; let stdout = String::from_utf8_lossy(&output.stdout).to_string(); let stderr = String::from_utf8_lossy(&output.stderr).to_string(); @@ -395,10 +574,7 @@ pub async fn extract_7z( } if !success { - let is_encrypted = SEVENZ_PASSWORD_PATTERNS - .iter() - .any(|p| combined.contains(p)); - if is_encrypted { + if sevenz_password_failure(&combined, &password_arg) { warn!( file = %archive_file.display(), "7z extraction failed — archive is password-protected" @@ -552,15 +728,22 @@ pub fn find_unrar() -> Option { None } -/// Find the 7z binary on the system. Checks `7z`, `7zz` (7-Zip standalone), -/// and `7za` (7-Zip standalone, older naming). +/// Find the 7z binary on the system. See [`pick_7z`] for the preference order. pub fn find_7z() -> Option { - for name in &["7z", "7zz", "7za"] { - if which_exists(name) { - return Some(name.to_string()); - } - } - None + pick_7z(which_exists) +} + +/// Pick the first available 7-Zip binary, given a predicate for "is on PATH". +/// +/// `7zz` (modern 7-Zip) comes first: on Debian/Ubuntu with both `7zip` and +/// `p7zip-full` installed, `7z` is p7zip 16.02, which reads passwords from +/// the terminal only, so the password would have to go in argv; `7zz` takes +/// it on stdin. Then `7z` (7-Zip or p7zip), then `7za` (older standalone). +fn pick_7z(exists: impl Fn(&str) -> bool) -> Option { + ["7zz", "7z", "7za"] + .into_iter() + .find(|name| exists(name)) + .map(str::to_string) } fn which_exists(name: &str) -> bool { @@ -579,6 +762,20 @@ mod tests { use std::fs; use std::io::Write; + /// On Debian with p7zip-full and 7zip both installed, `7z` is p7zip + /// (password from the terminal, so it has to go in argv) and `7zz` is + /// modern 7-Zip (password on stdin): prefer `7zz`. + #[test] + fn pick_7z_prefers_7zz_over_7z() { + let all = |_: &str| true; + assert_eq!(pick_7z(all).as_deref(), Some("7zz")); + let no_7zz = |name: &str| name != "7zz"; + assert_eq!(pick_7z(no_7zz).as_deref(), Some("7z")); + let only_7za = |name: &str| name == "7za"; + assert_eq!(pick_7z(only_7za).as_deref(), Some("7za")); + assert_eq!(pick_7z(|_: &str| false), None); + } + #[tokio::test] async fn test_extract_zip_valid() { let dir = tempfile::tempdir().unwrap(); @@ -844,9 +1041,10 @@ mod tests { #[test] fn sevenz_password_arg_is_omitted_without_password() { - assert_eq!(sevenz_password_arg(None), None); + assert_eq!(sevenz_password_arg(&PasswordArg::None), None); + assert_eq!(sevenz_password_arg(&PasswordArg::Stdin), None); assert_eq!( - sevenz_password_arg(Some("secret")).as_deref(), + sevenz_password_arg(&PasswordArg::Argv("secret".into())).as_deref(), Some("-psecret") ); } @@ -930,10 +1128,10 @@ mod tests { let rar = Path::new("/tmp/test.rar"); let out = Path::new("/tmp/out"); - let sevenz_args = rar_extract_args_with_7z(rar, out, None); + let sevenz_args = rar_extract_args_with_7z(rar, out, &PasswordArg::None); assert!(!sevenz_args.iter().any(|arg| arg == "-p-")); - let unrar_args = rar_extract_args_with_unrar(rar, out, None); + let unrar_args = rar_extract_args_with_unrar(rar, out, &PasswordArg::None); assert!(unrar_args.iter().any(|arg| arg == "-p-")); } @@ -942,10 +1140,10 @@ mod tests { let archive = Path::new("/tmp/test.7z"); let out = Path::new("/tmp/out"); - let args = sevenz_extract_args(archive, out, None); + let args = sevenz_extract_args(archive, out, &PasswordArg::None); assert!(!args.iter().any(|arg| arg == "-p-")); - let args = sevenz_extract_args(archive, out, Some("secret")); + let args = sevenz_extract_args(archive, out, &PasswordArg::Argv("secret".into())); assert!(args.iter().any(|arg| arg == "-psecret")); } @@ -954,4 +1152,257 @@ mod tests { let error: anyhow::Error = ArchivePasswordRequired.into(); assert!(error.downcast_ref::().is_some()); } + + // -- Archive passwords stay off the command line ---------------------- + + /// A stand-in extractor. Invoked with no arguments it prints `banner` + /// (the version probe); otherwise it runs `body`. `$secret_in_argv` is + /// `yes` when any argument contains the test password. + #[cfg(unix)] + fn fake_extractor(dir: &Path, name: &str, banner: &str, body: &str) -> PathBuf { + use std::os::unix::fs::PermissionsExt; + let path = dir.join(name); + fs::write( + &path, + format!( + "#!/bin/sh\nif [ $# -eq 0 ]; then printf '%s\\n' '{banner}'; exit 0; fi\n\ + secret_in_argv=no\nfor a; do case \"$a\" in *s3cr3t*) secret_in_argv=yes;; esac; out=$a; done\n\ + {body}\n" + ), + ) + .unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap(); + path + } + + /// Reads the password from stdin like unrar >= 6 / 7-Zip >= 21 do, and + /// fails if it was also passed as an argument. + #[cfg(unix)] + const STDIN_PASSWORD_BODY: &str = "[ \"$secret_in_argv\" = no ] || { echo 'password leaked into argv' >&2; exit 7; }\n\ + printf 'Enter password (will not be echoed): ' >&2\n\ + IFS= read -r pw\n\ + if [ \"$pw\" = 's3cr3t pw' ]; then mkdir -p \"$out\"; printf x > \"$out/movie.mkv\"; echo 'All OK'; exit 0; fi\n\ + echo 'Incorrect password for movie.mkv' >&2; exit 11"; + + #[cfg(unix)] + async fn run_rar( + unrar: &Path, + use_7z: bool, + password: Option<&str>, + ) -> anyhow::Result { + let work = tempfile::tempdir().unwrap(); + let rar = work.path().join("release.rar"); + fs::write(&rar, b"Rar!").unwrap(); + let out = work.path().join("out"); + tokio::time::timeout( + std::time::Duration::from_secs(20), + extract_rar_with(unrar.to_str().unwrap(), use_7z, &rar, &out, password), + ) + .await + .expect("extractor must not hang waiting for a password") + } + + #[cfg(unix)] + #[tokio::test] + async fn unrar_password_is_fed_on_stdin_not_argv() { + let tools = tempfile::tempdir().unwrap(); + let unrar = fake_extractor( + tools.path(), + "unrar", + "UNRAR 7.20 freeware Copyright (c) 1993-2025 Alexander Roshal", + STDIN_PASSWORD_BODY, + ); + let result = run_rar(&unrar, false, Some("s3cr3t pw")).await.unwrap(); + assert!(result.success, "{}", result.error_output); + assert_eq!(result.files_extracted.len(), 1); + } + + #[cfg(unix)] + #[tokio::test] + async fn sevenz_password_is_fed_on_stdin_not_argv() { + let tools = tempfile::tempdir().unwrap(); + let sevenz = fake_extractor( + tools.path(), + "7zz", + "7-Zip (z) 25.01 (arm64) : Copyright (c) 1999-2025 Igor Pavlov : 2025-08-03", + STDIN_PASSWORD_BODY, + ); + let result = run_rar(&sevenz, true, Some("s3cr3t pw")).await.unwrap(); + assert!(result.success, "{}", result.error_output); + } + + /// unrar 7 reports a missing or wrong password as `Incorrect password + /// for ...` with exit code 11; that must stay a typed error. + #[cfg(unix)] + #[tokio::test] + async fn wrong_password_on_stdin_is_typed_and_does_not_hang() { + let tools = tempfile::tempdir().unwrap(); + // Asks twice: the second read must see EOF, not block. + let body = "IFS= read -r pw\nIFS= read -r again\n\ + echo 'Incorrect password for movie.mkv' >&2; exit 11"; + let unrar = fake_extractor(tools.path(), "unrar", "UNRAR 7.20 freeware", body); + let error = run_rar(&unrar, false, Some("wrong")).await.unwrap_err(); + assert!( + error.downcast_ref::().is_some(), + "{error:#}" + ); + + let sevenz = fake_extractor( + tools.path(), + "7zz", + "7-Zip (z) 25.01 (arm64)", + "printf 'Enter password:' ; IFS= read -r pw\n\ + echo 'ERROR: Data Error in encrypted file. Wrong password? : movie.mkv' >&2; exit 2", + ); + let error = run_rar(&sevenz, true, Some("wrong")).await.unwrap_err(); + assert!( + error.downcast_ref::().is_some(), + "{error:#}" + ); + } + + /// The prompt we answer ourselves is not evidence of a password problem: + /// any other failure stays an ordinary failed extraction. + #[cfg(unix)] + #[tokio::test] + async fn answered_prompt_does_not_turn_other_failures_into_password_errors() { + let tools = tempfile::tempdir().unwrap(); + let body = "printf 'Enter password (will not be echoed): ' >&2\nIFS= read -r pw\n\ + echo 'movie.mkv - CRC failed' >&2; exit 3"; + let unrar = fake_extractor(tools.path(), "unrar", "UNRAR 7.20 freeware", body); + let result = run_rar(&unrar, false, Some("s3cr3t pw")).await.unwrap(); + assert!(!result.success); + } + + /// Extractors that are not known to read a piped password (old unrar, + /// unrar-free, p7zip) still get it as an argument. + #[cfg(unix)] + #[tokio::test] + async fn unknown_extractors_fall_back_to_the_password_argument() { + let tools = tempfile::tempdir().unwrap(); + let body = "[ \"$secret_in_argv\" = yes ] || exit 7\nmkdir -p \"$out\"; printf x > \"$out/movie.mkv\"; echo 'All OK'"; + for (name, banner) in [ + ( + "unrar", + "UNRAR 5.61 beta 1 freeware Copyright (c) 1993-2018 Alexander Roshal", + ), + ("unrar-free", "unrar-free 0.3.1"), + ( + "7z", + "7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov : 2016-05-21\np7zip Version 16.02", + ), + ] { + let bin = fake_extractor(tools.path(), name, banner, body); + let result = run_rar(&bin, name == "7z", Some("s3cr3t")).await.unwrap(); + assert!(result.success, "{name}: {}", result.error_output); + } + } + + #[test] + fn stdin_password_support_is_read_from_the_banner() { + for (banner, expected) in [ + ( + "UNRAR 7.20 beta 3 freeware Copyright (c) 1993-2025 Alexander Roshal", + true, + ), + ( + "UNRAR 6.21 freeware Copyright (c) 1993-2023 Alexander Roshal", + true, + ), + ("RAR 7.01 Copyright (c) 1993-2024 Alexander Roshal", true), + ("UNRAR 5.61 beta 1 freeware", false), + ("unrar-free 0.3.1", false), + ( + "7-Zip (z) 25.01 (arm64) : Copyright (c) 1999-2025 Igor Pavlov : 2025-08-03", + true, + ), + ( + "7-Zip 23.01 (x64) : Copyright (c) 1999-2023 Igor Pavlov : 2023-06-20", + true, + ), + ( + "7-Zip [64] 16.02 : Copyright (c) 1999-2016 Igor Pavlov\np7zip Version 16.02", + false, + ), + ("", false), + ] { + assert_eq!( + banner_reads_password_from_stdin(banner), + expected, + "{banner}" + ); + } + } + + #[test] + fn password_args_never_carry_the_password_in_stdin_mode() { + let rar = Path::new("/tmp/test.rar"); + let out = Path::new("/tmp/out"); + let stdin = PasswordArg::Stdin; + assert!( + rar_extract_args_with_unrar(rar, out, &stdin) + .iter() + .any(|a| a == "-p") + ); + assert!( + !rar_extract_args_with_7z(rar, out, &stdin) + .iter() + .any(|a| a.starts_with("-p")) + ); + assert!( + !sevenz_extract_args(rar, out, &stdin) + .iter() + .any(|a| a.starts_with("-p")) + ); + } + + /// End to end against a real 7z, when one is installed: an encrypted + /// archive (with encrypted headers) extracts with the right password and + /// reports a typed error for a wrong or missing one, without hanging. + #[tokio::test] + async fn real_7z_encrypted_archive_round_trip() { + let Some(sevenz) = find_7z() else { + eprintln!("skipping: no 7z/7zz/7za on PATH"); + return; + }; + let work = tempfile::tempdir().unwrap(); + fs::write(work.path().join("movie.mkv"), b"payload").unwrap(); + let archive = work.path().join("release.7z"); + let created = std::process::Command::new(&sevenz) + .current_dir(work.path()) + .args(["a", "-pp4ss word", "-mhe=on", "release.7z", "movie.mkv"]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(created.status.success(), "{created:?}"); + + let run = |password: Option<&'static str>, out: &'static str| { + let sevenz = sevenz.clone(); + let archive = archive.clone(); + let out = work.path().join(out); + async move { + tokio::time::timeout( + std::time::Duration::from_secs(60), + extract_7z_with(&sevenz, &archive, &out, password), + ) + .await + .expect("7z must not hang waiting for a password") + } + }; + + let ok = run(Some("p4ss word"), "ok").await.unwrap(); + assert!(ok.success, "{}", ok.error_output); + assert_eq!( + fs::read(work.path().join("ok/movie.mkv")).unwrap(), + b"payload" + ); + + for (password, out) in [(Some("wrong"), "wrong"), (None, "none")] { + let error = run(password, out).await.unwrap_err(); + assert!( + error.downcast_ref::().is_some(), + "{password:?}: {error:#}" + ); + } + } } diff --git a/crates/nzb-web/Cargo.toml b/crates/nzb-web/Cargo.toml index b71e168f..d67a66bb 100644 --- a/crates/nzb-web/Cargo.toml +++ b/crates/nzb-web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nzb-web" -version = "0.4.23" +version = "0.4.24" edition = "2024" description = "Usenet download engine: queue management, download orchestration, and background services" license = "MIT" @@ -56,6 +56,7 @@ all = { level = "warn", priority = -1 } unused = "warn" [dev-dependencies] +tokio = { version = "1", features = ["full", "test-util"] } tempfile = "3.27.0" nzb-nntp = { version = "0.2.24", path = "../nzb-nntp", features = ["test-support"] } yenc-simd = { version = "0.1" } diff --git a/crates/nzb-web/src/dir_watcher.rs b/crates/nzb-web/src/dir_watcher.rs index a27894fe..e804ae22 100644 --- a/crates/nzb-web/src/dir_watcher.rs +++ b/crates/nzb-web/src/dir_watcher.rs @@ -220,10 +220,19 @@ impl DirWatcher { // Archive entries may carry directories; the job is named after // the NZB itself, without its extension. let base = nzb_name.rsplit(['/', '\\']).next().unwrap_or(&nzb_name); - let name = base.strip_suffix(".nzb").unwrap_or(base); + // SABnzbd's `name{{password}}` convention: split the password + // off before parse_nzb sanitizes the name. + let (base, password) = match crate::nzb_core::path::split_job_password(base) { + Some((name, password)) => (name, Some(password).filter(|pw| !pw.is_empty())), + None => (base.to_string(), None), + }; + let name = base.strip_suffix(".nzb").unwrap_or(&base); let name = if name.is_empty() { "unknown" } else { name }; let mut job = crate::nzb_core::nzb_parser::parse_nzb(name, &data) .map_err(|e| format!("parse failed for {nzb_name}: {e}"))?; + if password.is_some() { + job.password = password; + } job.work_dir = self.queue_manager.incomplete_dir().join(&job.id); job.output_dir = self.queue_manager.complete_dir().join(&job.name); std::fs::create_dir_all(&job.work_dir) diff --git a/crates/nzb-web/src/direct_unpack.rs b/crates/nzb-web/src/direct_unpack.rs index 04e2e652..e31479d4 100644 --- a/crates/nzb-web/src/direct_unpack.rs +++ b/crates/nzb-web/src/direct_unpack.rs @@ -19,7 +19,9 @@ use tokio::sync::Notify; use tokio::task::JoinHandle; use tracing::{debug, error, info, warn}; -use nzb_postproc::{find_unrar, normalize_extracted_permissions}; +use nzb_postproc::{ + extractor_reads_password_from_stdin, find_unrar, normalize_extracted_permissions, +}; /// Error strings from unrar output that indicate an unrecoverable failure. const UNRAR_ERROR_PATTERNS: &[&str] = &[ @@ -37,6 +39,14 @@ const UNRAR_ERROR_PATTERNS: &[&str] = &[ "Unexpected end of archive", ]; +/// The subset of [`UNRAR_ERROR_PATTERNS`] that means a missing or wrong +/// archive password. +const UNRAR_PASSWORD_PATTERNS: &[&str] = &[ + "password is incorrect", + "Incorrect password", + "in the encrypted file", +]; + /// A healthy extraction can wait for volumes, but a process that neither exits /// nor emits output is stuck (for example, waiting on an unrecognised prompt). const DIRECT_UNPACK_IDLE_TIMEOUT: Duration = Duration::from_secs(300); @@ -47,6 +57,9 @@ pub struct DirectUnpackResult { pub set_name: String, pub success: bool, pub error: Option, + /// The failure was a missing or wrong archive password, the condition + /// the batch extractor reports as `archive_password_required`. + pub password_required: bool, } /// Tracks which volumes are available for a single RAR set. @@ -274,12 +287,20 @@ fn unpack_set( set_name: set_name.to_string(), success: false, error: Some(format!("Failed to create output dir: {e}")), + password_required: false, }; } - let pw_flag = match password { - Some(pw) => format!("-p{pw}"), - None => "-p-".to_string(), + // unrar 6+ reads the password from stdin after a bare `-p`, which keeps it + // out of `ps` and /proc//cmdline. Older builds get `-p`. A + // password with a line break cannot be sent as one line. + let password = password.filter(|pw| !pw.is_empty()); + let stdin_password = password + .filter(|pw| !pw.contains(['\n', '\r']) && extractor_reads_password_from_stdin(unrar_bin)); + let pw_flag = match (password, stdin_password) { + (None, _) => "-p-".to_string(), + (Some(_), Some(_)) => "-p".to_string(), + (Some(pw), None) => format!("-p{pw}"), }; // Spawn unrar with -vp (pause between volumes). @@ -300,11 +321,30 @@ fn unpack_set( set_name: set_name.to_string(), success: false, error: Some(format!("Failed to spawn unrar: {e}")), + password_required: false, }; } }; - let result = drive_unrar(&mut child, set_name, state, volume_ready, killed, rt); + // Answer the password prompt before anything else; unrar asks for it + // first. Stdin stays open afterwards for the volume prompts. + if let (Some(pw), Some(stdin)) = (stdin_password, child.stdin.as_mut()) + && let Err(e) = stdin + .write_all(format!("{pw}\n").as_bytes()) + .and_then(|()| stdin.flush()) + { + warn!(set = %set_name, error = %e, "Failed to write the archive password to unrar"); + } + + let result = drive_unrar( + &mut child, + set_name, + usize::from(stdin_password.is_some()), + state, + volume_ready, + killed, + rt, + ); // Ensure child is cleaned up. let _ = child.kill(); @@ -325,6 +365,7 @@ fn unpack_set( fn drive_unrar( child: &mut Child, set_name: &str, + answered_password_prompts: usize, state: &Mutex, volume_ready: &Notify, killed: &AtomicBool, @@ -338,6 +379,9 @@ fn drive_unrar( spawn_output_reader(stderr, output_tx); let mut next_volume: u32 = 1; // Volume 0 is already being processed. + // The password prompt we answered up front appears in the first output + // only; any further prompt means the password was rejected. + let mut answered_password_prompts = answered_password_prompts; let mut output_buf = String::with_capacity(1024); let mut last_output = Instant::now(); @@ -359,6 +403,7 @@ fn drive_unrar( set_name: set_name.to_string(), success: true, error: None, + password_required: false, }; } @@ -367,7 +412,16 @@ fn drive_unrar( .find(|pattern| output_buf.contains(**pattern)) { error!(set = %set_name, error = %output_buf.trim(), "Direct unpack error detected"); - return failed_result(set_name, &format!("unrar reported {pattern}")); + let mut result = failed_result(set_name, &format!("unrar reported {pattern}")); + result.password_required = UNRAR_PASSWORD_PATTERNS.contains(pattern); + return result; + } + + if output_buf.matches("Enter password").count() > answered_password_prompts { + warn!(set = %set_name, "Unrar asked for the archive password again — aborting"); + let mut result = failed_result(set_name, "unrar rejected the archive password"); + result.password_required = true; + return result; } if is_volume_prompt(&output_buf) { @@ -381,6 +435,7 @@ fn drive_unrar( let _ = stdin.flush(); next_volume += 1; output_buf.clear(); + answered_password_prompts = 0; } Err(e) => { let _ = stdin.write_all(b"Q\n"); @@ -452,6 +507,7 @@ fn failed_result(set_name: &str, error: &str) -> DirectUnpackResult { set_name: set_name.to_string(), success: false, error: Some(error.to_string()), + password_required: false, } } @@ -461,9 +517,13 @@ fn result_from_exit_status(set_name: &str, status: std::process::ExitStatus) -> set_name: set_name.to_string(), success: true, error: None, + password_required: false, } } else { - failed_result(set_name, &format!("unrar exited with status {status}")) + let mut result = failed_result(set_name, &format!("unrar exited with status {status}")); + // unrar exit code 11 is RARX_BADPWD: missing or wrong password. + result.password_required = status.code() == Some(11); + result } } @@ -678,6 +738,77 @@ mod tests { assert_eq!(result.error.as_deref(), Some("unrar reported Cannot open")); } + /// Runs `unpack_set` on a one-volume set with `unrar` and `password`. + #[cfg(unix)] + fn unpack_one_volume(script: &str, password: Option<&str>) -> DirectUnpackResult { + let (script_dir, unrar) = fake_unrar(script); + let output_dir = tempfile::tempdir().unwrap(); + let first_volume = script_dir.path().join("movie.part001.rar"); + std::fs::write(&first_volume, b"first").unwrap(); + let state = Mutex::new(DirectUnpackState { + sets: BTreeMap::from([( + "movie".to_string(), + RarSetState { + set_name: "movie".to_string(), + volumes: BTreeMap::from([(0, first_volume.clone())]), + }, + )]), + download_finished: true, + }); + let runtime = tokio::runtime::Runtime::new().unwrap(); + unpack_set( + unrar.to_str().unwrap(), + "movie", + &first_volume, + output_dir.path(), + password, + &state, + &Notify::new(), + &AtomicBool::new(false), + runtime.handle(), + ) + } + + /// A stand-in for unrar >= 6: prints its banner when probed with no + /// arguments, refuses a password on the command line and reads it from + /// stdin after `-p`. + #[cfg(unix)] + const STDIN_PASSWORD_UNRAR: &str = "#!/bin/sh\n\ + if [ $# -eq 0 ]; then echo 'UNRAR 7.20 freeware'; exit 0; fi\n\ + for a; do case \"$a\" in *s3cr3t*) echo 'password leaked into argv' >&2; exit 7;; esac; done\n\ + printf 'Enter password (will not be echoed): ' >&2\n\ + IFS= read -r pw\n\ + if [ \"$pw\" = 's3cr3t' ]; then printf 'All OK\\n'; exit 0; fi\n\ + printf 'Incorrect password for movie.mkv\\n' >&2\n\ + IFS= read -r again\nexit 11\n"; + + #[cfg(unix)] + #[test] + fn direct_unpack_feeds_the_password_on_stdin() { + let result = unpack_one_volume(STDIN_PASSWORD_UNRAR, Some("s3cr3t")); + assert!(result.success, "{result:?}"); + assert!(!result.password_required); + } + + #[cfg(unix)] + #[test] + fn direct_unpack_wrong_password_is_typed() { + let result = unpack_one_volume(STDIN_PASSWORD_UNRAR, Some("wrong")); + assert!(!result.success); + assert!(result.password_required, "{result:?}"); + } + + #[cfg(unix)] + #[test] + fn direct_unpack_other_errors_are_not_password_errors() { + let result = unpack_one_volume( + "#!/bin/sh\nprintf 'Cannot open archive\\n' >&2\nexit 2\n", + None, + ); + assert!(!result.success); + assert!(!result.password_required); + } + #[test] fn test_direct_unpack_state_basics() { let state = DirectUnpackState { diff --git a/crates/nzb-web/src/error.rs b/crates/nzb-web/src/error.rs index ef077b36..2fc8a01b 100644 --- a/crates/nzb-web/src/error.rs +++ b/crates/nzb-web/src/error.rs @@ -67,6 +67,15 @@ impl ApiError { } } + /// 409 for a create that collides with an existing entry, such as a + /// category, server or feed that is already configured. + pub fn conflict(msg: impl Into) -> Self { + Self { + status: Some(StatusCode::CONFLICT), + kind: ApiErrorKind::Message(msg.into()), + } + } + pub const fn admission_conflict() -> Self { Self { status: Some(StatusCode::CONFLICT), @@ -74,6 +83,25 @@ impl ApiError { } } + /// A caller-supplied URL that the server-side fetch guard refuses + /// (bad scheme, malformed, or a private/reserved/metadata destination). + /// This is client input, so it is a 400 with `error_kind: url_rejected`. + pub fn url_rejected(msg: impl Into) -> Self { + Self { + status: Some(StatusCode::BAD_REQUEST), + kind: ApiErrorKind::UrlRejected(msg.into()), + } + } + + /// A server-side fetch reached the remote host but it failed (error + /// status, unreachable host): 502 with `error_kind: bad_gateway`. + pub fn bad_gateway(msg: impl Into) -> Self { + Self { + status: Some(StatusCode::BAD_GATEWAY), + kind: ApiErrorKind::Message(msg.into()), + } + } + pub fn status(&self) -> StatusCode { self.status.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR) } @@ -93,6 +121,8 @@ pub enum ApiErrorKind { Text(&'static str), #[error("{0}")] Message(String), + #[error("{0}")] + UrlRejected(String), #[error(transparent)] Anyhow(#[from] anyhow::Error), #[error(transparent)] @@ -136,12 +166,14 @@ impl Serialize for ApiError { | ApiErrorKind::Core(crate::nzb_core::NzbError::AdmissionConflict) => { "admission_conflict" } + ApiErrorKind::UrlRejected(_) => "url_rejected", // Otherwise classify by status so clients can tell a bad // request or missing resource from a server fault. _ => match self.status() { StatusCode::BAD_REQUEST => "bad_request", StatusCode::NOT_FOUND => "not_found", StatusCode::CONFLICT => "conflict", + StatusCode::BAD_GATEWAY => "bad_gateway", _ => "internal_error", }, }, @@ -161,6 +193,17 @@ impl From for ApiError { } } +impl From for ApiError { + /// A multipart body that cannot be read is the client's fault: axum + /// reports 400 for a malformed body and 413 for one over the limit. + fn from(e: axum::extract::multipart::MultipartError) -> Self { + Self { + status: Some(e.status()), + kind: ApiErrorKind::Message(format!("Multipart error: {}", e.body_text())), + } + } +} + impl From for ApiError { /// Map domain errors to the HTTP status that describes them: missing /// resources are 404, malformed input 400, conflicts 409, and only @@ -252,5 +295,24 @@ mod tests { assert_eq!(json["error_kind"], "not_found"); let json = serde_json::to_value(ApiError::bad_request("bad")).unwrap(); assert_eq!(json["error_kind"], "bad_request"); + let json = serde_json::to_value(ApiError::conflict("taken")).unwrap(); + assert_eq!(json["error_kind"], "conflict"); + assert_eq!(json["status"], 409); + } + + #[test] + fn fetch_errors_map_to_400_and_502() { + let rejected = ApiError::url_rejected("URL scheme 'file' not allowed"); + assert_eq!(rejected.status(), StatusCode::BAD_REQUEST); + let json = serde_json::to_value(&rejected).unwrap(); + assert_eq!(json["error_kind"], "url_rejected"); + assert_eq!(json["status"], 400); + assert_eq!(json["human_readable"], "URL scheme 'file' not allowed"); + + let upstream = ApiError::bad_gateway("URL returned HTTP 500"); + assert_eq!(upstream.status(), StatusCode::BAD_GATEWAY); + let json = serde_json::to_value(&upstream).unwrap(); + assert_eq!(json["error_kind"], "bad_gateway"); + assert_eq!(json["status"], 502); } } diff --git a/crates/nzb-web/src/fetch_guard.rs b/crates/nzb-web/src/fetch_guard.rs index 9a002fc2..3bb203dd 100644 --- a/crates/nzb-web/src/fetch_guard.rs +++ b/crates/nzb-web/src/fetch_guard.rs @@ -223,7 +223,7 @@ enum GuardError { } async fn validate_inner(raw_url: &str, policy: &FetchPolicy) -> Result { - let reject = |e: anyhow::Error| GuardError::Rejected(ApiError::from(e)); + let reject = |e: anyhow::Error| GuardError::Rejected(ApiError::url_rejected(e.to_string())); let url = reqwest::Url::parse(raw_url).map_err(|e| reject(anyhow::anyhow!("Invalid URL: {e}")))?; @@ -269,14 +269,14 @@ async fn validate_inner(raw_url: &str, policy: &FetchPolicy) -> Result = tokio::net::lookup_host(format!("{host}:{port}")) .await .map_err(|e| { - GuardError::Unresolved(ApiError::from(anyhow::anyhow!( + GuardError::Unresolved(ApiError::bad_gateway(format!( "DNS resolution failed for '{host}': {e}" ))) })? .collect(); if addrs.is_empty() { - return Err(GuardError::Unresolved(ApiError::from(anyhow::anyhow!( + return Err(GuardError::Unresolved(ApiError::bad_gateway(format!( "DNS resolution returned no addresses for '{host}'" )))); } @@ -402,6 +402,29 @@ fn is_globally_routable(ip: IpAddr) -> bool { mod tests { use super::*; + #[tokio::test] + async fn guard_rejections_are_400_url_rejected() { + for url in [ + "http://127.0.0.1/file.nzb", + "http://localhost/file.nzb", + "http://10.0.0.1/file.nzb", + "http://169.254.169.254/latest/meta-data/", + "file:///etc/passwd", + "not a url", + "http://", + ] { + let err = validate_fetch_url(url).await.unwrap_err(); + assert_eq!(err.status(), http::StatusCode::BAD_REQUEST, "{url}"); + let json = serde_json::to_value(&err).unwrap(); + assert_eq!(json["error_kind"], "url_rejected", "{url}"); + + let err = check_fetch_url_allowed(url, &FetchPolicy::strict()) + .await + .unwrap_err(); + assert_eq!(err.status(), http::StatusCode::BAD_REQUEST, "{url}"); + } + } + #[tokio::test] async fn validate_fetch_url_rejects_private_ip_literals() { let err = validate_fetch_url("http://127.0.0.1/file.nzb") diff --git a/crates/nzb-web/src/lib.rs b/crates/nzb-web/src/lib.rs index e5625d45..9adec749 100644 --- a/crates/nzb-web/src/lib.rs +++ b/crates/nzb-web/src/lib.rs @@ -21,7 +21,8 @@ pub mod util; pub use log_buffer::{LogBuffer, LogBufferLayer}; pub use nzb_dispatch::{ArticleFailure, ArticleFailureKind}; pub use queue_manager::{ - DailyStatisticsData, GlobalStatisticsData, QueueManager, ServerStatsData, StatisticsPeriodData, + DailyStatisticsData, GlobalStatisticsData, HistoryRetryOutcome, QueueManager, QueueSortField, + ServerStatsData, StatisticsPeriodData, }; pub use startup::{StartupConfig, StartupResult}; pub use state::AppState; diff --git a/crates/nzb-web/src/queue_manager.rs b/crates/nzb-web/src/queue_manager.rs index 00e69747..cfd2279d 100644 --- a/crates/nzb-web/src/queue_manager.rs +++ b/crates/nzb-web/src/queue_manager.rs @@ -23,7 +23,7 @@ use crate::nzb_core::models::*; use crate::nzb_core::nzb_parser; use nzb_postproc::{ PostProcConfig, PostProcLimits, PostProcResourcePool, PostProcResourceSnapshot, - has_usable_output, parse_rar_volume, run_pipeline_with_cleanup, + has_usable_output, parse_rar_volume, run_pipeline_with_cleanup, run_repair_pipeline, }; use crate::direct_unpack::DirectUnpacker; @@ -1081,6 +1081,83 @@ struct JobState { output_dir_claimed: bool, } +/// Field a manual queue sort orders by. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum QueueSortField { + /// Fraction of the job's bytes still to download. + Remaining, + /// Job name, case-insensitively. + Name, + /// Total job size in bytes. + Size, + /// Time since the job was added; ascending is youngest first, matching + /// SABnzbd's `avg_age`. + Age, + /// Job priority, `Low` first when ascending. + Priority, +} + +impl QueueSortField { + fn compare(self, left: &NzbJob, right: &NzbJob) -> std::cmp::Ordering { + match self { + Self::Remaining => { + let remaining = |job: &NzbJob| { + let total = job.total_bytes.max(1); + (total.saturating_sub(job.downloaded_bytes), total) + }; + let (left_remaining, left_total) = remaining(left); + let (right_remaining, right_total) = remaining(right); + (left_remaining as u128 * right_total as u128) + .cmp(&(right_remaining as u128 * left_total as u128)) + } + Self::Name => left + .name + .chars() + .flat_map(char::to_lowercase) + .cmp(right.name.chars().flat_map(char::to_lowercase)), + Self::Size => left.total_bytes.cmp(&right.total_bytes), + // Younger means added later. + Self::Age => right.added_at.cmp(&left.added_at), + Self::Priority => (left.priority as u8).cmp(&(right.priority as u8)), + } + } +} + +impl std::str::FromStr for QueueSortField { + type Err = (); + + /// Parse a sort field name, case-insensitively. Accepts SABnzbd's + /// `avg_age` and `bytes` spellings alongside the native names. + fn from_str(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "remaining" => Ok(Self::Remaining), + "name" => Ok(Self::Name), + "size" | "bytes" => Ok(Self::Size), + "age" | "avg_age" => Ok(Self::Age), + "priority" => Ok(Self::Priority), + _ => Err(()), + } + } +} + +/// Why a job, or the whole queue, is paused. +/// +/// Exposed on the native API so a pause applied by the disk-space guard can +/// be told apart from one the user asked for. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PauseReason { + /// The user paused this job. + Manual, + /// The user paused all downloads (Pause All or a timed pause). + Global, + /// The disk-space guard (`min_free_space_bytes`) is holding downloads. + /// The hold lifts automatically once space frees up. + DiskSpace, + /// The job was paused after a news-server error. + Server, +} + /// Notification fired immediately when a job is accepted into the queue. #[derive(Debug, Clone)] pub struct JobAddedEvent { @@ -1095,6 +1172,18 @@ pub struct JobAddedEvent { // QueueManager // --------------------------------------------------------------------------- +/// Result of [`QueueManager::retry_history_entry`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HistoryRetryOutcome { + /// A new job was enqueued; carries its id. + Started(String), + /// A retry of this entry is already queued or running; carries that + /// job's id. Nothing new was enqueued. + InProgress(String), + /// The history entry has no stored NZB to retry from. + NoNzbData, +} + /// Thread-safe queue manager that coordinates all downloads. /// /// Wrapped in `Arc` for sharing between the background task and HTTP handlers. @@ -1115,6 +1204,10 @@ pub struct QueueManager { /// Keeping this separate from ordinary per-job pause state means that /// `resume_all` only resumes work stopped by `pause_all`. globally_paused_jobs: Mutex>, + /// Set while the current global pause was applied by the disk-space + /// guard rather than the user; holds when the guard tripped. The guard + /// lifts only pauses it owns. + disk_space_hold: Mutex>>, /// Global speed tracker. speed: SpeedTracker, /// Database for persistence. @@ -1135,6 +1228,10 @@ pub struct QueueManager { add_tx: broadcast::Sender, /// Max concurrent active downloads (0 = unlimited). max_active_downloads: AtomicUsize, + /// History retries in flight: history entry id -> id of the job the + /// retry enqueued. Held from the in-flight check through the enqueue so + /// concurrent retries of one entry admit exactly one job. + history_retries: Mutex>, /// Automatically keep the queue ordered by remaining percentage. auto_sort_remaining_pct: AtomicBool, /// Optional bounded post-processing hooks. @@ -1167,6 +1264,7 @@ pub struct QueueManager { impl QueueManager { const GLOBAL_PAUSED_JOBS_SETTING: &'static str = "globally_paused_job_ids"; + const DISK_SPACE_HOLD_SETTING: &'static str = "disk_space_hold_since"; /// Create a new queue manager. #[allow(clippy::too_many_arguments)] @@ -1250,6 +1348,7 @@ impl QueueManager { globally_paused: AtomicBool::new(false), pause_transition: Mutex::new(()), globally_paused_jobs: Mutex::new(HashSet::new()), + disk_space_hold: Mutex::new(None), speed: SpeedTracker::new(), db: Mutex::new(db), incomplete_dir: Mutex::new(incomplete_dir), @@ -1260,6 +1359,7 @@ impl QueueManager { log_buffer: Some(log_buffer), add_tx, max_active_downloads: AtomicUsize::new(max_active_downloads), + history_retries: Mutex::new(HashMap::new()), auto_sort_remaining_pct: AtomicBool::new(false), postproc_scripts: Mutex::new(PostProcScriptConfig::default()), postproc_resources: PostProcResourcePool::new(postproc_limits), @@ -1283,11 +1383,17 @@ impl QueueManager { /// Post-processing level for a job: its own override (e.g. SABnzbd's /// `pp`), else its category's setting, else repair+unpack (3). pub fn post_processing_level(&self, job: &NzbJob) -> u8 { - job.pp_override.unwrap_or_else(|| { + self.post_processing_level_for(job.pp_override, &job.category) + } + + /// [`Self::post_processing_level`] from the job's fields, for callers + /// that hold a borrow of another part of the job. + fn post_processing_level_for(&self, pp_override: Option, category: &str) -> u8 { + pp_override.unwrap_or_else(|| { self.categories .lock() .iter() - .find(|c| c.name == job.category) + .find(|c| c.name == category) .map(|c| c.post_processing) .unwrap_or(3) // default: repair+unpack }) @@ -1464,19 +1570,24 @@ impl QueueManager { /// queue order is retained for equal percentages, which keeps repeated /// manual sorts deterministic and avoids active-job churn. pub fn sort_by_remaining_percentage(&self, ascending: bool) { + self.sort_queue(QueueSortField::Remaining, ascending); + } + + /// Stable sort of the queue by `field`. Only the queue order changes: + /// job status is untouched, so active downloads keep running, and the + /// original order is retained for equal keys, which keeps repeated + /// manual sorts deterministic and avoids active-job churn. + pub fn sort_queue(&self, field: QueueSortField, ascending: bool) { let jobs = self.jobs.lock(); let mut order = self.job_order.lock(); order.sort_by(|left, right| { - let remaining = |id: &String| { - jobs.get(id).map_or((0u64, 1u64), |state| { - let total = state.job.total_bytes.max(1); - (total.saturating_sub(state.job.downloaded_bytes), total) - }) + let left = jobs.get(left).map(|state| &state.job); + let right = jobs.get(right).map(|state| &state.job); + let ordering = match (left, right) { + (Some(left), Some(right)) => field.compare(left, right), + // Ids without a job state keep their relative position. + _ => std::cmp::Ordering::Equal, }; - let (left_remaining, left_total) = remaining(left); - let (right_remaining, right_total) = remaining(right); - let ordering = (left_remaining as u128 * right_total as u128) - .cmp(&(right_remaining as u128 * left_total as u128)); if ascending { ordering } else { @@ -1781,6 +1892,49 @@ impl QueueManager { self.start_next_queued(); } + /// Retry one history entry at most once at a time. + /// + /// The first caller claims the entry, rebuilds the job (reusing a + /// retained partial work directory via [`Self::prepare_retry_job`]) and + /// enqueues it. While that job is still in the queue -- queued, paused, + /// downloading or post-processing -- any further retry of the same entry + /// returns [`HistoryRetryOutcome::InProgress`] with the existing job id + /// instead of enqueueing a duplicate. Once the job has left the queue the + /// entry can be retried again. + /// + /// This is an in-flight guard, not the durable `Idempotency-Key` + /// admission: a history entry stays retryable after its retry finishes, + /// whereas an admission key binds exactly one job forever. + pub fn retry_history_entry( + self: &Arc, + entry: &HistoryEntry, + ) -> crate::nzb_core::Result { + let mut retries = self.history_retries.lock(); + { + let jobs = self.jobs.lock(); + retries.retain(|_, job_id| jobs.contains_key(job_id)); + if let Some(job_id) = retries.get(&entry.id) { + return Ok(HistoryRetryOutcome::InProgress(job_id.clone())); + } + } + + let Some(nzb_data) = self.history_get_nzb_data(&entry.id)? else { + return Ok(HistoryRetryOutcome::NoNzbData); + }; + let retry_data = self.history_get_retry_data(&entry.id)?; + let job = self.prepare_retry_job(entry, &nzb_data, retry_data.as_deref())?; + let job_id = job.id.clone(); + info!( + name = %job.name, + id = %job_id, + original_id = %entry.id, + "Retrying NZB from history" + ); + self.add_job(job, Some(nzb_data))?; + retries.insert(entry.id.clone(), job_id.clone()); + Ok(HistoryRetryOutcome::Started(job_id)) + } + /// Rebuild a history job for retry. Newer history rows carry a checkpoint /// and retain a partial work directory when at least one article was /// written, so the dispatcher can enqueue only unresolved articles and @@ -2155,6 +2309,12 @@ impl QueueManager { // unpacker so extraction overlaps with download. if self.direct_unpack_enabled.load(Ordering::Relaxed) && state.job.articles_failed == 0 + // Download-only and repair-only jobs + // deliver their archives unpacked. + && self.post_processing_level_for( + state.job.pp_override, + &state.job.category, + ) >= 2 && let Some(vol_info) = parse_rar_volume(&file.filename) { if state.direct_unpacker.is_none() { @@ -2511,6 +2671,7 @@ impl QueueManager { direct_unpacker, password, content_articles_failed, + delete_archives, ) = { let mut jobs = self.jobs.lock(); let Some(state) = jobs.get_mut(job_id) else { @@ -2545,6 +2706,7 @@ impl QueueManager { du, pw, content_failed, + state.job.delete_archives.unwrap_or(true), ) }; // Record the reserved folder so a restart resumes into it. @@ -2603,6 +2765,7 @@ impl QueueManager { job_id = %job_id, set = %r.set_name, error = ?r.error, + password_required = r.password_required, "Direct unpack failed for set — falling back to normal extraction" ); } @@ -2636,6 +2799,7 @@ impl QueueManager { .unwrap_or_default(); let config = PostProcConfig { cleanup_after_extract: true, + delete_archives, output_dir: Some(output_dir.clone()), articles_failed, content_articles_failed, @@ -2644,14 +2808,19 @@ impl QueueManager { max_nested_archive_depth: self.max_nested_archive_depth, }; - let result = run_pipeline_with_cleanup( - &work_dir, - &config, - Some(&self.postproc_resources), - &cleanup_patterns, - &unwanted_extensions, - ) - .await; + let result = if pp_level >= 2 { + run_pipeline_with_cleanup( + &work_dir, + &config, + Some(&self.postproc_resources), + &cleanup_patterns, + &unwanted_extensions, + ) + .await + } else { + // Repair only (SABnzbd `pp=1`): the archives are the output. + run_repair_pipeline(&work_dir, &config, Some(&self.postproc_resources)).await + }; info!( job_id = %job_id, @@ -2676,18 +2845,22 @@ impl QueueManager { result.stages } else { - info!(job_id = %job_id, pp_level, "Post-processing disabled for category, skipping pipeline"); - // No pipeline to repair — if articles failed, mark as failed now - if !success { - let mut jobs = self.jobs.lock(); - if let Some(state) = jobs.get_mut(job_id) { - state.job.status = JobStatus::Failed; - state.job.error_message = - Some(format!("{articles_failed} article(s) failed to download")); - state.failure_code = Some(JobFailureCode::ArticlesUnavailable); - } + info!(job_id = %job_id, pp_level, "Post-processing disabled for job, skipping pipeline"); + // Download only (SABnzbd `pp=0`): nothing is verified or + // repaired, so the raw files are delivered as downloaded, as + // SABnzbd does. Hopeless jobs never reach this point. + if success { + Vec::new() + } else { + vec![StageResult { + name: "Verify".to_string(), + status: StageStatus::Skipped, + message: Some(format!( + "Post-processing disabled; {articles_failed} article(s) failed to download" + )), + duration_secs: 0.0, + }] } - Vec::new() }; // Run the configured hook after the final status is known. Its output @@ -2877,7 +3050,10 @@ impl QueueManager { // A post-processing job that contains only raw archive/PAR2 artifacts // is not a usable completion. Check before moving residual work files // so a bad job cannot pollute the completed directory. - if final_status == JobStatus::Completed && !stages.is_empty() { + // Only unpacking jobs promise extracted output: download-only (0) and + // repair-only (1) jobs deliver the archives themselves. + let pp_level = self.post_processing_level(&state.job); + if final_status == JobStatus::Completed && pp_level >= 2 && !stages.is_empty() { let output_has_payload = has_usable_output(&state.job.output_dir).unwrap_or(false); let work_has_payload = has_usable_output(&state.job.work_dir).unwrap_or(false); if !output_has_payload && !work_has_payload { @@ -2984,6 +3160,8 @@ impl QueueManager { error_message: state.job.error_message.clone(), failure_code: (final_status == JobStatus::Failed) .then(|| state.failure_code.unwrap_or(JobFailureCode::DownloadFailed)), + post_processing: Some(pp_level), + delete_archives: state.job.delete_archives, server_stats: state.job.server_stats.clone(), nzb_data: state.nzb_data.clone(), retry_data, @@ -3295,6 +3473,12 @@ impl QueueManager { .ok_or_else(|| crate::nzb_core::NzbError::JobNotFound(id.to_string()))?; state.job.status = JobStatus::Paused; + // An explicit pause supersedes a disk-space hold, so the disk + // guard must not resume this job when space frees up. + if state.failure_code == Some(JobFailureCode::StorageUnavailable) { + state.failure_code = None; + state.job.error_message = None; + } let db = self.db.lock(); db.queue_update_progress( @@ -3461,6 +3645,8 @@ impl QueueManager { failure_code: Some( state.failure_code.unwrap_or(JobFailureCode::DownloadFailed), ), + post_processing: Some(self.post_processing_level(&state.job)), + delete_archives: state.job.delete_archives, server_stats: state.job.server_stats.clone(), nzb_data: state.nzb_data.clone(), retry_data: None, @@ -3568,12 +3754,20 @@ impl QueueManager { /// Pause all downloads globally. pub fn pause_all(&self) { + self.pause_all_with_hold(None); + } + + /// Pause all downloads. `disk_space_hold` marks a pause applied by the + /// disk-space guard; `None` is a user pause, which also takes ownership + /// of any existing disk hold so the guard will not lift it. + fn pause_all_with_hold(&self, disk_space_hold: Option>) { let _transition = self.pause_transition.lock(); // Publish the global gate before touching individual jobs. This // prevents every scheduling and per-job resume path from starting // more work while the active contexts are being paused. self.globally_paused.store(true, Ordering::SeqCst); self.db.lock().set_setting("globally_paused", "true"); + self.set_disk_space_hold(disk_space_hold); // A plain pause is indefinite: cancel any pending timed resume. // `pause_for` sets its deadline after calling this. *self.pause_until.lock() = None; @@ -3657,6 +3851,7 @@ impl QueueManager { let _transition = self.pause_transition.lock(); self.globally_paused.store(false, Ordering::SeqCst); self.db.lock().set_setting("globally_paused", "false"); + self.set_disk_space_hold(None); *self.pause_until.lock() = None; let paused_by_global = std::mem::take(&mut *self.globally_paused_jobs.lock()); @@ -3776,6 +3971,11 @@ impl QueueManager { .collect() } + /// Whether a job with exactly this id is in the queue. + pub fn contains_job(&self, job_id: &str) -> bool { + self.jobs.lock().contains_key(job_id) + } + /// Get a single job by ID (with files included). pub fn get_job(&self, job_id: &str) -> Option { let jobs = self.jobs.lock(); @@ -3796,6 +3996,112 @@ impl QueueManager { self.globally_paused.load(Ordering::SeqCst) } + /// Why downloads are globally paused, or `None` when they are not. + pub fn global_pause_reason(&self) -> Option { + if !self.is_paused() { + return None; + } + Some(if self.disk_space_hold.lock().is_some() { + PauseReason::DiskSpace + } else { + PauseReason::Global + }) + } + + /// When the disk-space guard paused downloads, if it is holding them now. + pub fn disk_space_hold_since(&self) -> Option> { + *self.disk_space_hold.lock() + } + + fn set_disk_space_hold(&self, since: Option>) { + *self.disk_space_hold.lock() = since; + let value = since.map(|at| at.to_rfc3339()).unwrap_or_default(); + self.db + .lock() + .set_setting(Self::DISK_SPACE_HOLD_SETTING, &value); + } + + /// Why a queued job is paused, or `None` when it is not paused. + pub fn pause_reason(&self, job_id: &str) -> Option { + let (storage_hold, server_error) = { + let jobs = self.jobs.lock(); + let state = jobs.get(job_id)?; + if state.job.status != JobStatus::Paused { + return None; + } + ( + state.failure_code == Some(JobFailureCode::StorageUnavailable), + state.job.error_message.is_some(), + ) + }; + if storage_hold { + return Some(PauseReason::DiskSpace); + } + if self.globally_paused_jobs.lock().contains(job_id) { + return Some(self.global_pause_reason().unwrap_or(PauseReason::Global)); + } + Some(if server_error { + PauseReason::Server + } else { + PauseReason::Manual + }) + } + + /// Run the disk-space guard once: pause all downloads when a configured + /// storage volume is below `min_free_space`, and lift the guard's own + /// holds once every volume has room again. User pauses are never lifted. + pub fn enforce_disk_guard(self: &Arc) { + let threshold = self.min_free_space(); + let paths = self.disk_guard_paths(); + let path_refs: Vec<_> = paths.iter().map(std::path::PathBuf::as_path).collect(); + if !disk_space_available(threshold, &path_refs) { + if !self.is_paused() { + warn!( + free_bytes = paths.first().map_or(0, |path| get_disk_free(path)), + min_free_space = threshold, + "Low disk space on a configured storage volume, auto-pausing downloads" + ); + self.pause_all_with_hold(Some(Utc::now())); + } + return; + } + + if self.is_paused() && self.disk_space_hold.lock().is_some() { + info!("Disk space recovered, resuming downloads paused by the disk guard"); + self.resume_all(); + } + self.resume_storage_held_jobs(); + } + + /// Requeue jobs the start-of-download disk check paused, once space is back. + fn resume_storage_held_jobs(self: &Arc) { + let _transition = self.pause_transition.lock(); + if self.is_paused() { + return; + } + let mut resumed = 0u32; + { + let mut jobs = self.jobs.lock(); + for state in jobs.values_mut() { + if state.job.status == JobStatus::Paused + && state.failure_code == Some(JobFailureCode::StorageUnavailable) + { + state.job.error_message = None; + state.failure_code = None; + state.job.status = JobStatus::Queued; + resumed += 1; + } + } + } + if resumed > 0 { + info!( + count = resumed, + "Resumed jobs held for low disk space now that space is available" + ); + self.start_next_queued(); + } + } + /// Get the number of jobs in the queue. pub fn queue_size(&self) -> usize { self.jobs @@ -3850,14 +4156,10 @@ impl QueueManager { .iter() .find(|configured| configured.name == category); if let Some(base) = category_config.and_then(|configured| configured.output_dir.as_ref()) { - let root = if base.is_absolute() { - base.clone() - } else { - crate::nzb_core::path::safe_join(&self.complete_dir(), &base.to_string_lossy()) - .ok_or_else(|| { - crate::nzb_core::NzbError::Other("category output path is unsafe".into()) - })? - }; + let root = crate::nzb_core::path::category_output_root(&self.complete_dir(), base) + .ok_or_else(|| { + crate::nzb_core::NzbError::Other("category output path is unsafe".into()) + })?; return crate::nzb_core::path::safe_join(&root, name).ok_or_else(|| { crate::nzb_core::NzbError::Other("category output path is unsafe".into()) }); @@ -4464,7 +4766,17 @@ impl QueueManager { }; if was_paused { self.globally_paused.store(true, Ordering::SeqCst); - info!("Restored global pause state from database"); + let hold = self + .db + .lock() + .get_setting(Self::DISK_SPACE_HOLD_SETTING) + .and_then(|value| DateTime::parse_from_rfc3339(&value).ok()) + .map(|at| at.with_timezone(&Utc)); + *self.disk_space_hold.lock() = hold; + info!( + disk_space_hold = hold.is_some(), + "Restored global pause state from database" + ); } // Reclaim partial downloads nothing can reach any more before any @@ -4863,20 +5175,8 @@ impl QueueManager { info!(total_nntp_connections = total, "NNTP connection summary"); } } - if tick_count.is_multiple_of(30) && qm.min_free_space() > 0 { - let paths = qm.disk_guard_paths(); - let path_refs: Vec<_> = paths.iter().map(std::path::PathBuf::as_path).collect(); - let free = paths.first().map_or(0, |path| get_disk_free(path)); - if !disk_space_available(qm.min_free_space(), &path_refs) - && !qm.globally_paused.load(Ordering::Relaxed) - { - warn!( - free_bytes = free, - min_free_space = qm.min_free_space(), - "Low disk space on a configured storage volume, auto-pausing downloads" - ); - qm.pause_all(); - } + if tick_count.is_multiple_of(30) { + qm.enforce_disk_guard(); } } }); @@ -4909,6 +5209,7 @@ mod global_pause_tests { error_message: None, speed_bps: 0, pp_override: None, + delete_archives: None, server_stats: Vec::new(), files: Vec::new(), } @@ -5045,6 +5346,98 @@ mod global_pause_tests { ); } + fn order_of(manager: &QueueManager) -> Vec { + manager.job_order.lock().clone() + } + + fn sort_fixture(manager: &QueueManager, root: &std::path::Path) { + let base = Utc::now(); + // (id, name, total, downloaded, priority, age in minutes, status) + for (id, name, total, downloaded, priority, age_min, status) in [ + ( + "a", + "charlie", + 300, + 0, + Priority::Normal, + 10, + JobStatus::Downloading, + ), + ("b", "Alpha", 100, 0, Priority::High, 30, JobStatus::Queued), + ("c", "bravo", 200, 0, Priority::Low, 20, JobStatus::Queued), + ] { + let mut j = job(id, status, root); + j.name = name.to_string(); + j.total_bytes = total; + j.downloaded_bytes = downloaded; + j.priority = priority; + j.added_at = base - chrono::Duration::minutes(age_min); + insert_job(manager, j); + } + } + + #[tokio::test] + async fn queue_sort_by_name_is_case_insensitive() { + let (manager, tempdir) = manager(); + sort_fixture(&manager, tempdir.path()); + manager.sort_queue(QueueSortField::Name, true); + assert_eq!(order_of(&manager), vec!["b", "c", "a"]); + manager.sort_queue(QueueSortField::Name, false); + assert_eq!(order_of(&manager), vec!["a", "c", "b"]); + // Sorting reorders only: the active job keeps downloading. + assert_eq!(manager.get_job("a").unwrap().status, JobStatus::Downloading); + } + + #[tokio::test] + async fn queue_sort_by_size_age_and_priority() { + let (manager, tempdir) = manager(); + sort_fixture(&manager, tempdir.path()); + manager.sort_queue(QueueSortField::Size, true); + assert_eq!(order_of(&manager), vec!["b", "c", "a"]); + manager.sort_queue(QueueSortField::Size, false); + assert_eq!(order_of(&manager), vec!["a", "c", "b"]); + // Age ascending is youngest first, like SABnzbd's avg_age. + manager.sort_queue(QueueSortField::Age, true); + assert_eq!(order_of(&manager), vec!["a", "c", "b"]); + manager.sort_queue(QueueSortField::Age, false); + assert_eq!(order_of(&manager), vec!["b", "c", "a"]); + manager.sort_queue(QueueSortField::Priority, true); + assert_eq!(order_of(&manager), vec!["c", "a", "b"]); + manager.sort_queue(QueueSortField::Priority, false); + assert_eq!(order_of(&manager), vec!["b", "a", "c"]); + } + + #[tokio::test] + async fn queue_sort_is_stable_for_equal_keys() { + let (manager, tempdir) = manager(); + for id in ["x", "y", "z"] { + let mut j = job(id, JobStatus::Queued, tempdir.path()); + j.name = "Same".into(); + insert_job(&manager, j); + } + manager.sort_queue(QueueSortField::Name, true); + assert_eq!(order_of(&manager), vec!["x", "y", "z"]); + manager.sort_queue(QueueSortField::Name, false); + assert_eq!(order_of(&manager), vec!["x", "y", "z"]); + } + + #[test] + fn queue_sort_field_parses_known_names_only() { + for (input, expected) in [ + ("remaining", QueueSortField::Remaining), + ("NAME", QueueSortField::Name), + ("size", QueueSortField::Size), + ("bytes", QueueSortField::Size), + ("age", QueueSortField::Age), + ("avg_age", QueueSortField::Age), + ("priority", QueueSortField::Priority), + ] { + assert_eq!(input.parse::(), Ok(expected), "{input}"); + } + assert!("bogus".parse::().is_err()); + assert!("".parse::().is_err()); + } + #[tokio::test] async fn speed_limit_above_u32_saturates_instead_of_wrapping() { let (manager, _tempdir) = manager(); @@ -5104,6 +5497,8 @@ mod global_pause_tests { stages: Vec::new(), error_message: None, failure_code: None, + post_processing: None, + delete_archives: None, server_stats: Vec::new(), nzb_data: None, retry_data: None, @@ -5221,6 +5616,143 @@ mod global_pause_tests { assert_eq!(manager.get_job("manual").unwrap().status, JobStatus::Paused); } + #[tokio::test] + async fn low_disk_guard_reports_disk_space_reason_and_resumes_when_space_frees() { + let (manager, tempdir) = manager(); + insert_job( + &manager, + job("active", JobStatus::Downloading, tempdir.path()), + ); + insert_job(&manager, job("manual", JobStatus::Paused, tempdir.path())); + assert_eq!(manager.pause_reason("active"), None); + assert_eq!(manager.pause_reason("manual"), Some(PauseReason::Manual)); + + // No volume can ever have u64::MAX bytes free. + manager.set_min_free_space(u64::MAX); + manager.enforce_disk_guard(); + + assert!(manager.is_paused()); + assert_eq!(manager.global_pause_reason(), Some(PauseReason::DiskSpace)); + assert!(manager.disk_space_hold_since().is_some()); + assert_eq!(manager.pause_reason("active"), Some(PauseReason::DiskSpace)); + assert_eq!(manager.pause_reason("manual"), Some(PauseReason::Manual)); + assert!( + manager + .db + .lock() + .get_setting(QueueManager::DISK_SPACE_HOLD_SETTING) + .is_some_and(|value| !value.is_empty()) + ); + + // Space frees up: the guard lifts its own hold on the next check. + manager.set_min_free_space(1); + manager.enforce_disk_guard(); + + assert!(!manager.is_paused()); + assert_eq!(manager.global_pause_reason(), None); + assert!(manager.disk_space_hold_since().is_none()); + assert_ne!(manager.get_job("active").unwrap().status, JobStatus::Paused); + assert_eq!(manager.get_job("manual").unwrap().status, JobStatus::Paused); + assert_eq!(manager.pause_reason("manual"), Some(PauseReason::Manual)); + } + + #[tokio::test] + async fn disk_guard_never_lifts_a_user_global_pause() { + let (manager, tempdir) = manager(); + insert_job(&manager, job("queued", JobStatus::Queued, tempdir.path())); + + manager.pause_all(); + assert_eq!(manager.global_pause_reason(), Some(PauseReason::Global)); + assert_eq!(manager.pause_reason("queued"), Some(PauseReason::Global)); + manager.set_min_free_space(1); + manager.enforce_disk_guard(); + assert!(manager.is_paused()); + + // A user Pause All during a disk hold takes ownership of the pause. + manager.resume_all(); + manager.set_min_free_space(u64::MAX); + manager.enforce_disk_guard(); + assert_eq!(manager.global_pause_reason(), Some(PauseReason::DiskSpace)); + manager.pause_all(); + assert_eq!(manager.global_pause_reason(), Some(PauseReason::Global)); + manager.set_min_free_space(1); + manager.enforce_disk_guard(); + assert!(manager.is_paused()); + } + + #[tokio::test] + async fn preflight_storage_hold_reports_disk_space_and_resumes_when_space_frees() { + let (manager, tempdir) = manager(); + insert_job(&manager, job("held", JobStatus::Paused, tempdir.path())); + { + let mut jobs = manager.jobs.lock(); + let state = jobs.get_mut("held").unwrap(); + state.job.error_message = Some("Paused: low disk space".to_string()); + state.failure_code = Some(JobFailureCode::StorageUnavailable); + } + assert_eq!(manager.pause_reason("held"), Some(PauseReason::DiskSpace)); + + manager.set_min_free_space(1); + manager.enforce_disk_guard(); + + let held = manager.get_job("held").unwrap(); + assert_ne!(held.status, JobStatus::Paused); + assert_eq!(held.error_message, None); + assert_eq!(manager.pause_reason("held"), None); + } + + #[tokio::test] + async fn manual_pause_takes_over_a_preflight_storage_hold() { + let (manager, tempdir) = manager(); + insert_job(&manager, job("held", JobStatus::Paused, tempdir.path())); + { + let mut jobs = manager.jobs.lock(); + let state = jobs.get_mut("held").unwrap(); + state.job.error_message = Some("Paused: low disk space".to_string()); + state.failure_code = Some(JobFailureCode::StorageUnavailable); + } + + manager.pause_job("held").unwrap(); + assert_eq!(manager.pause_reason("held"), Some(PauseReason::Manual)); + + manager.set_min_free_space(1); + manager.enforce_disk_guard(); + assert_eq!(manager.get_job("held").unwrap().status, JobStatus::Paused); + } + + #[tokio::test] + async fn server_error_pause_reports_server_reason() { + let (manager, tempdir) = manager(); + insert_job(&manager, job("srv", JobStatus::Paused, tempdir.path())); + manager + .jobs + .lock() + .get_mut("srv") + .unwrap() + .job + .error_message = Some("server unavailable".to_string()); + assert_eq!(manager.pause_reason("srv"), Some(PauseReason::Server)); + } + + #[tokio::test] + async fn restored_disk_space_hold_keeps_its_reason() { + let (manager, _tempdir) = manager(); + { + let db = manager.db.lock(); + db.set_setting("globally_paused", "true"); + db.set_setting( + QueueManager::DISK_SPACE_HOLD_SETTING, + &Utc::now().to_rfc3339(), + ); + } + manager.restore_from_db().unwrap(); + assert_eq!(manager.global_pause_reason(), Some(PauseReason::DiskSpace)); + + manager.set_min_free_space(1); + manager.enforce_disk_guard(); + assert!(!manager.is_paused()); + } + #[tokio::test] async fn provider_waiting_status_keeps_job_active_and_clears_on_recovery() { let (manager, tempdir) = manager(); @@ -5542,6 +6074,187 @@ mod global_pause_tests { ); } + fn set_default_pp(manager: &QueueManager, level: u8) { + manager.set_categories(vec![CategoryConfig { + post_processing: level, + ..CategoryConfig::default() + }]); + } + + fn write_raw_artifacts(work_dir: &std::path::Path) { + std::fs::create_dir_all(work_dir).unwrap(); + std::fs::write(work_dir.join("release.part001.rar"), b"raw").unwrap(); + std::fs::write(work_dir.join("release.part002.rar"), b"raw").unwrap(); + std::fs::write(work_dir.join("release.par2"), b"par2").unwrap(); + } + + /// SABnzbd `pp=0` is download only: the raw archives are the output, so + /// the "no usable output" rule must not fail the job (even when a + /// post-processing script contributed a stage). + #[tokio::test] + async fn download_only_job_with_raw_archives_is_completed() { + let (manager, tempdir) = manager(); + set_default_pp(&manager, 0); + let raw = job("download-only", JobStatus::PostProcessing, tempdir.path()); + write_raw_artifacts(&raw.work_dir); + let output = raw.output_dir.clone(); + insert_job(&manager, raw); + + let stages = vec![StageResult { + name: "Script".to_string(), + status: StageStatus::Success, + message: Some("ok".to_string()), + duration_secs: 0.0, + }]; + let mut jobs = manager.jobs.lock(); + manager.move_to_history(jobs.get_mut("download-only").unwrap(), stages); + drop(jobs); + + let entry = manager + .db + .lock() + .history_get("download-only") + .unwrap() + .unwrap(); + assert_eq!( + entry.status, + JobStatus::Completed, + "{:?}", + entry.error_message + ); + assert_eq!(entry.post_processing, Some(0)); + assert!(output.join("release.part001.rar").exists()); + assert!(output.join("release.par2").exists()); + } + + /// SABnzbd `pp=1` is repair only: archives are the expected output. + #[tokio::test] + async fn repair_only_job_with_raw_archives_is_completed() { + let (manager, tempdir) = manager(); + set_default_pp(&manager, 1); + let raw = job("repair-only", JobStatus::PostProcessing, tempdir.path()); + write_raw_artifacts(&raw.work_dir); + let output = raw.output_dir.clone(); + insert_job(&manager, raw); + + let stages = vec![StageResult { + name: "Verify".to_string(), + status: StageStatus::Skipped, + message: Some("Skipped".to_string()), + duration_secs: 0.0, + }]; + let mut jobs = manager.jobs.lock(); + manager.move_to_history(jobs.get_mut("repair-only").unwrap(), stages); + drop(jobs); + + let entry = manager + .db + .lock() + .history_get("repair-only") + .unwrap() + .unwrap(); + assert_eq!( + entry.status, + JobStatus::Completed, + "{:?}", + entry.error_message + ); + assert_eq!(entry.post_processing, Some(1)); + assert!(output.join("release.part002.rar").exists()); + } + + /// The per-job override decides the level recorded in history. + #[tokio::test] + async fn history_records_the_job_override_level() { + let (manager, tempdir) = manager(); + set_default_pp(&manager, 3); + let mut raw = job("override-level", JobStatus::PostProcessing, tempdir.path()); + raw.pp_override = Some(1); + write_raw_artifacts(&raw.work_dir); + insert_job(&manager, raw); + + let mut jobs = manager.jobs.lock(); + manager.move_to_history(jobs.get_mut("override-level").unwrap(), Vec::new()); + drop(jobs); + + let entry = manager + .db + .lock() + .history_get("override-level") + .unwrap() + .unwrap(); + assert_eq!(entry.post_processing, Some(1)); + assert_eq!(entry.status, JobStatus::Completed); + } + + /// `pp=1` must not unpack: a (bogus) archive is delivered as-is instead + /// of failing extraction. + #[tokio::test] + async fn repair_only_pipeline_does_not_extract() { + let (manager, tempdir) = manager(); + set_default_pp(&manager, 1); + let raw = job( + "repair-no-unpack", + JobStatus::PostProcessing, + tempdir.path(), + ); + std::fs::create_dir_all(&raw.work_dir).unwrap(); + std::fs::write(raw.work_dir.join("release.zip"), b"not really a zip").unwrap(); + let output = raw.output_dir.clone(); + insert_job(&manager, raw); + + manager.on_job_finished("repair-no-unpack", true, 0).await; + + let entry = manager + .db + .lock() + .history_get("repair-no-unpack") + .unwrap() + .unwrap(); + assert_eq!( + entry.status, + JobStatus::Completed, + "{:?}", + entry.error_message + ); + assert!(entry.stages.iter().all(|stage| stage.name != "Extract")); + assert!(output.join("release.zip").exists()); + } + + /// `pp=0` does not verify or repair, so missing articles do not fail + /// the job; the raw files are delivered like SABnzbd does. + #[tokio::test] + async fn download_only_job_with_failed_articles_is_completed() { + let (manager, tempdir) = manager(); + set_default_pp(&manager, 0); + let raw = job( + "download-only-damaged", + JobStatus::PostProcessing, + tempdir.path(), + ); + write_raw_artifacts(&raw.work_dir); + let output = raw.output_dir.clone(); + insert_job(&manager, raw); + + manager + .on_job_finished("download-only-damaged", false, 2) + .await; + + let entry = manager + .db + .lock() + .history_get("download-only-damaged") + .unwrap() + .unwrap(); + assert_eq!( + entry.status, + JobStatus::Completed, + "{:?}", + entry.error_message + ); + assert!(output.join("release.part001.rar").exists()); + } + #[tokio::test] async fn post_processing_with_payload_is_completed() { let (manager, tempdir) = manager(); diff --git a/crates/nzb-web/src/rss_monitor.rs b/crates/nzb-web/src/rss_monitor.rs index aa0b81bf..26351722 100644 --- a/crates/nzb-web/src/rss_monitor.rs +++ b/crates/nzb-web/src/rss_monitor.rs @@ -1,9 +1,11 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::sync::Arc; use arc_swap::ArcSwap; use chrono::Utc; +use tokio::sync::Notify; +use tokio::time::{Duration, Instant}; use tracing::{info, warn}; use crate::fetch_guard::{ @@ -22,6 +24,9 @@ pub struct RssMonitor { config: Arc>, queue_manager: Arc, data_dir: PathBuf, + /// Woken whenever the config changes, so feed additions, edits and + /// deletions are picked up without waiting for the current sleep. + wake: Arc, } impl RssMonitor { @@ -34,9 +39,17 @@ impl RssMonitor { config, queue_manager, data_dir, + wake: Arc::new(Notify::new()), } } + /// Use `wake` as the config-change signal (see + /// [`crate::state::AppState::rss_monitor_wake`]). + pub fn with_wake(mut self, wake: Arc) -> Self { + self.wake = wake; + self + } + /// Migrate legacy rss_seen.json entries into the database on first run. fn migrate_seen_json(&self) { let seen_file = self.data_dir.join("rss_seen.json"); @@ -80,58 +93,75 @@ impl RssMonitor { info!("Legacy rss_seen.json migrated and removed"); } - /// Run the monitor loop forever, polling feeds at their configured intervals. - /// Reads feed config from the shared ArcSwap on each iteration so that - /// feeds added/removed/toggled via the API take effect without a restart. + /// Run the monitor loop forever, polling each feed at its own configured + /// interval. Feed config is re-read from the shared ArcSwap on every pass, + /// and the loop sleeps only until the next feed is due or the config + /// changes (see [`RssMonitor::with_wake`]), so feeds added, edited, + /// re-enabled or removed via the API take effect at once. pub async fn run(self) { info!("RSS monitor started"); // Migrate legacy seen file on first run self.migrate_seen_json(); - loop { - let cfg = self.config.load(); - let feeds = &cfg.rss_feeds; + let mut schedule = FeedSchedule::default(); + let mut next_maintenance = Instant::now(); - for feed in feeds { - if !feed.enabled { - continue; - } + loop { + // A full Arc rather than an arc-swap guard: it is held across the + // feed checks below. + let cfg = self.config.load_full(); + schedule.sync(&cfg.rss_feeds); + + let now = Instant::now(); + let due: Vec<&RssFeedConfig> = cfg + .rss_feeds + .iter() + .filter(|feed| feed.enabled && schedule.is_due(feed, now)) + .collect(); + for feed in &due { if let Err(e) = self.check_feed(feed).await { warn!(feed = %feed.name, error = %e, "RSS feed check failed"); } + schedule.mark_checked(feed, Instant::now()); } - // Prune old items based on config - let limit = cfg.general.rss_history_limit.unwrap_or(500); - if let Ok(count) = self.queue_manager.rss_item_count() - && count > limit - && let Ok(pruned) = self.queue_manager.rss_items_prune(limit) - && pruned > 0 - { - info!(pruned, "Pruned old RSS items"); + if !due.is_empty() || Instant::now() >= next_maintenance { + self.prune_items(&cfg); + next_maintenance = Instant::now() + IDLE_INTERVAL; } - if let Some(days) = cfg.general.rss_downloaded_item_expiry_days { - let cutoff = (Utc::now() - chrono::Duration::days(days as i64)).to_rfc3339(); - if let Ok(expired) = self.queue_manager.rss_items_expire_downloaded(&cutoff) - && expired > 0 - { - info!(expired, "Expired downloaded RSS items"); - } + let wake_at = schedule + .next_due(&cfg.rss_feeds) + .map_or(next_maintenance, |due| due.min(next_maintenance)); + drop(cfg); + + tokio::select! { + _ = tokio::time::sleep_until(wake_at) => {} + _ = self.wake.notified() => {} } + } + } - // Use the minimum poll interval across all enabled feeds, defaulting to 15 min - let interval = feeds - .iter() - .filter(|f| f.enabled) - .map(|f| f.poll_interval_secs) - .min() - .unwrap_or(900); + /// Trim the stored RSS item history to the configured limits. + fn prune_items(&self, cfg: &AppConfig) { + let limit = cfg.general.rss_history_limit.unwrap_or(500); + if let Ok(count) = self.queue_manager.rss_item_count() + && count > limit + && let Ok(pruned) = self.queue_manager.rss_items_prune(limit) + && pruned > 0 + { + info!(pruned, "Pruned old RSS items"); + } - drop(cfg); - tokio::time::sleep(tokio::time::Duration::from_secs(interval)).await; + if let Some(days) = cfg.general.rss_downloaded_item_expiry_days { + let cutoff = (Utc::now() - chrono::Duration::days(days as i64)).to_rfc3339(); + if let Ok(expired) = self.queue_manager.rss_items_expire_downloaded(&cutoff) + && expired > 0 + { + info!(expired, "Expired downloaded RSS items"); + } } } @@ -165,13 +195,16 @@ impl RssMonitor { ), }; - // Load download rules for this feed + // Load download rules for this feed (an empty feed list means all feeds) let rules = self .queue_manager .rss_rule_list() .unwrap_or_default() .into_iter() - .filter(|r| r.enabled && r.feed_names.iter().any(|n| n == &feed.name)) + .filter(|r| { + r.enabled + && (r.feed_names.is_empty() || r.feed_names.iter().any(|n| n == &feed.name)) + }) .collect::>(); // Collect all items for batch insert (single DB lock) @@ -405,6 +438,95 @@ impl RssMonitor { } } +/// How long the monitor sleeps when no feed is due sooner; also the cadence +/// of RSS item history maintenance. +const IDLE_INTERVAL: Duration = Duration::from_secs(900); + +/// Smallest accepted `poll_interval_secs`. The API rejects anything lower, +/// and the monitor clamps lower values from existing or hand-edited configs +/// so a `0` interval cannot spin the loop against the indexer. +pub const MIN_POLL_INTERVAL_SECS: u64 = 60; + +/// A feed's poll interval with [`MIN_POLL_INTERVAL_SECS`] applied. +fn effective_poll_interval(feed: &RssFeedConfig) -> Duration { + Duration::from_secs(feed.poll_interval_secs.max(MIN_POLL_INTERVAL_SECS)) +} + +/// When each enabled feed was last checked, keyed by feed name. +/// +/// A feed's next due time is derived from its *current* `poll_interval_secs` +/// on every pass, so an interval edit takes effect immediately. Feeds that are +/// unknown (newly added, renamed, re-enabled or re-pointed at a new URL) are +/// due at once; deleted or disabled feeds are forgotten. +#[derive(Debug, Default)] +struct FeedSchedule { + last_checked: HashMap, + /// Feeds already warned about for an interval below the floor. + clamped: HashSet, +} + +impl FeedSchedule { + /// Drop entries for feeds that are gone, disabled, or now point elsewhere, + /// and warn (once per feed) about intervals below the floor. Returns the + /// names newly warned about. + fn sync(&mut self, feeds: &[RssFeedConfig]) -> Vec { + self.last_checked.retain(|name, (url, _)| { + feeds + .iter() + .any(|f| f.enabled && &f.name == name && &f.url == url) + }); + + let too_fast: HashSet<&str> = feeds + .iter() + .filter(|f| f.enabled && f.poll_interval_secs < MIN_POLL_INTERVAL_SECS) + .map(|f| f.name.as_str()) + .collect(); + self.clamped.retain(|name| too_fast.contains(name.as_str())); + let mut warned = Vec::new(); + for feed in feeds { + if too_fast.contains(feed.name.as_str()) && self.clamped.insert(feed.name.clone()) { + warn!( + feed = %feed.name, + poll_interval_secs = feed.poll_interval_secs, + min_secs = MIN_POLL_INTERVAL_SECS, + "RSS feed poll interval is below the minimum; using the minimum" + ); + warned.push(feed.name.clone()); + } + } + warned + } + + fn due_at(&self, feed: &RssFeedConfig) -> Option { + self.last_checked.get(&feed.name).map(|(_, at)| { + // An absurd interval must not overflow `Instant`; a year out + // is effectively "never" for a poller. + at.checked_add(effective_poll_interval(feed)) + .unwrap_or(*at + Duration::from_secs(365 * 86_400)) + }) + } + + fn is_due(&self, feed: &RssFeedConfig, now: Instant) -> bool { + self.due_at(feed).is_none_or(|due| due <= now) + } + + fn mark_checked(&mut self, feed: &RssFeedConfig, at: Instant) { + self.last_checked + .insert(feed.name.clone(), (feed.url.clone(), at)); + } + + /// Earliest due time across enabled feeds; `None` when there are none. + /// A never-checked feed counts as due now. + fn next_due(&self, feeds: &[RssFeedConfig]) -> Option { + let now = Instant::now(); + feeds + .iter() + .filter(|f| f.enabled) + .map(|f| self.due_at(f).unwrap_or(now)) + .min() + } +} + #[cfg(test)] mod tests { use super::*; @@ -487,6 +609,104 @@ mod tests { assert!(RssMonitor::compile_filter(&"x".repeat(513)).is_none()); } + /// Serve a one-item feed plus its NZB on loopback, and return a monitor + /// whose fetch policy admits that loopback address. + async fn monitor_with_local_feed( + data_dir: PathBuf, + ) -> (RssMonitor, Arc, RssFeedConfig) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback"); + let base = format!("http://{}", listener.local_addr().expect("local addr")); + let feed_xml = format!( + r#" + feedFeed2026-07-27T00:00:00Z + new-itemShow.S01E01.1080p + 2026-07-27T00:00:00Z + + + "# + ); + let nzb = r#" + + + alt.binaries.test + seg@example.test + +"#; + let app = axum::Router::new() + .route( + "/feed.xml", + axum::routing::get(move || async move { feed_xml }), + ) + .route( + "/release.nzb", + axum::routing::get(move || async move { nzb }), + ); + tokio::spawn(async move { axum::serve(listener, app).await }); + + let (mut monitor, queue_manager) = monitor(data_dir); + let mut config = AppConfig::default(); + config.general.fetch_allowed_hosts = vec!["127.0.0.1".into()]; + monitor.config = Arc::new(ArcSwap::from_pointee(config)); + let feed = RssFeedConfig { + name: "tv-feed".into(), + url: format!("{base}/feed.xml"), + poll_interval_secs: 900, + category: None, + filter_regex: None, + enabled: true, + auto_download: false, + max_age_days: None, + }; + (monitor, queue_manager, feed) + } + + fn rule(feed_names: Vec) -> crate::nzb_core::models::RssRule { + crate::nzb_core::models::RssRule { + id: "rule".into(), + name: "TV".into(), + feed_names, + category: None, + priority: 1, + match_regex: "1080p".into(), + enabled: true, + } + } + + #[tokio::test] + async fn rule_with_no_feed_names_applies_to_every_feed() { + let temp = tempfile::tempdir().expect("tempdir"); + let (monitor, queue_manager, feed) = monitor_with_local_feed(temp.path().into()).await; + queue_manager + .rss_rule_insert(&rule(Vec::new())) + .expect("insert rule"); + + monitor.check_feed(&feed).await.expect("feed check"); + + assert_eq!(queue_manager.get_jobs().len(), 1); + assert!( + queue_manager + .rss_item_get("new-item") + .expect("database query") + .expect("stored item") + .downloaded + ); + } + + #[tokio::test] + async fn rule_naming_another_feed_does_not_apply() { + let temp = tempfile::tempdir().expect("tempdir"); + let (monitor, queue_manager, feed) = monitor_with_local_feed(temp.path().into()).await; + queue_manager + .rss_rule_insert(&rule(vec!["other-feed".into()])) + .expect("insert rule"); + + monitor.check_feed(&feed).await.expect("feed check"); + + assert!(queue_manager.get_jobs().is_empty()); + } + #[tokio::test] async fn feed_checks_reject_local_targets_before_network_access() { let temp = tempfile::tempdir().expect("tempdir"); @@ -508,4 +728,257 @@ mod tests { .expect_err("local addresses must be rejected"); assert!(error.to_string().contains("private/reserved")); } + + const EMPTY_RSS: &str = + r#"t"#; + + /// Serve `EMPTY_RSS` on loopback and report the (tokio, so possibly + /// paused) time of every request that arrives. + async fn feed_server() -> ( + std::net::SocketAddr, + tokio::sync::mpsc::UnboundedReceiver, + ) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind feed server"); + let addr = listener.local_addr().expect("feed server addr"); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let mut request = Vec::new(); + let mut buf = [0u8; 1024]; + while !request.windows(4).any(|w| w == b"\r\n\r\n") { + match socket.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => request.extend_from_slice(&buf[..n]), + } + } + let _ = tx.send(tokio::time::Instant::now()); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/rss+xml\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + EMPTY_RSS.len(), + EMPTY_RSS + ); + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.shutdown().await; + } + }); + (addr, rx) + } + + fn feed(name: &str, addr: std::net::SocketAddr, poll_interval_secs: u64) -> RssFeedConfig { + RssFeedConfig { + name: name.into(), + url: format!("http://{addr}/{name}.xml"), + poll_interval_secs, + category: None, + filter_regex: None, + enabled: true, + auto_download: false, + max_age_days: None, + } + } + + /// BUG-66: with no feeds at boot the monitor sleeps for the 900 s idle + /// interval, and a feed added through the API used to wait out that whole + /// sleep before its first check. Saving the config must wake the monitor. + /// Runs on a paused clock, so the 900 s sleep costs no wall-clock time and + /// the assertion is on virtual time. + #[tokio::test(start_paused = true)] + async fn feed_config_changes_take_effect_without_waiting_for_the_current_sleep() { + let temp = tempfile::tempdir().expect("tempdir"); + let (addr, mut requests) = feed_server().await; + + let mut config = AppConfig::default(); + config.general.fetch_allowed_hosts = vec!["127.0.0.1".into()]; + let (monitor, queue_manager) = monitor(temp.path().to_path_buf()); + let shared = Arc::clone(&monitor.config); + shared.store(Arc::new(config)); + let state = crate::state::AppState::new( + Arc::clone(&shared), + temp.path().join("config.toml"), + queue_manager, + LogBuffer::default(), + Arc::new(crate::auth::TokenStore::new()), + Arc::new(crate::auth::CredentialStore::new(temp.path().to_path_buf())), + ); + let monitor = monitor.with_wake(Arc::clone(&state.rss_monitor_wake)); + let task = tokio::spawn(monitor.run()); + + // Let the monitor run its empty first pass and go to sleep. + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + + let added_at = tokio::time::Instant::now(); + state + .update_config_with(|config| { + config.rss_feeds.push(feed("late", addr, 600)); + Ok::<_, anyhow::Error>(()) + }) + .expect("add feed"); + + let fetched_at = requests.recv().await.expect("feed was fetched"); + let waited = fetched_at - added_at; + assert!( + waited < std::time::Duration::from_secs(5), + "new feed waited {waited:?} for its first check" + ); + + // Shortening the interval takes effect without waiting out the old 600 s. + state + .update_config_with(|config| { + config.rss_feeds[0].poll_interval_secs = 60; + Ok::<_, anyhow::Error>(()) + }) + .expect("edit feed"); + let refetched_at = requests.recv().await.expect("feed was re-fetched"); + let gap = refetched_at - fetched_at; + assert!( + gap < std::time::Duration::from_secs(120), + "edited interval took {gap:?} to apply" + ); + + // A deleted feed is never fetched again. + state + .update_config_with(|config| { + config.rss_feeds.clear(); + Ok::<_, anyhow::Error>(()) + }) + .expect("delete feed"); + tokio::time::sleep(std::time::Duration::from_secs(3600)).await; + while let Ok(at) = requests.try_recv() { + assert!( + at <= refetched_at + std::time::Duration::from_secs(1), + "deleted feed was fetched again after {:?}", + at - refetched_at + ); + } + task.abort(); + } + + fn schedule_feed(name: &str, url: &str, poll_interval_secs: u64) -> RssFeedConfig { + RssFeedConfig { + name: name.into(), + url: url.into(), + poll_interval_secs, + category: None, + filter_regex: None, + enabled: true, + auto_download: false, + max_age_days: None, + } + } + + #[test] + fn schedule_checks_new_feeds_at_once_and_then_per_feed_interval() { + let t0 = Instant::now(); + let fast = schedule_feed("fast", "https://example.test/fast", 60); + let slow = schedule_feed("slow", "https://example.test/slow", 3600); + let feeds = vec![fast.clone(), slow.clone()]; + let mut schedule = FeedSchedule::default(); + + schedule.sync(&feeds); + assert!(schedule.is_due(&fast, t0) && schedule.is_due(&slow, t0)); + + schedule.mark_checked(&fast, t0); + schedule.mark_checked(&slow, t0); + let at_61 = t0 + Duration::from_secs(61); + assert!(schedule.is_due(&fast, at_61)); + assert!( + !schedule.is_due(&slow, at_61), + "slow feed is not over-polled" + ); + assert_eq!( + schedule.next_due(&feeds), + Some(t0 + Duration::from_secs(60)) + ); + } + + #[test] + fn schedule_applies_interval_edits_immediately() { + let t0 = Instant::now(); + let mut feed = schedule_feed("f", "https://example.test/f", 900); + let mut schedule = FeedSchedule::default(); + schedule.mark_checked(&feed, t0); + assert_eq!(schedule.due_at(&feed), Some(t0 + Duration::from_secs(900))); + + feed.poll_interval_secs = 120; + schedule.sync(std::slice::from_ref(&feed)); + assert_eq!(schedule.due_at(&feed), Some(t0 + Duration::from_secs(120))); + + feed.poll_interval_secs = u64::MAX; + assert!( + schedule.due_at(&feed).is_some(), + "huge interval must not overflow" + ); + } + + #[test] + fn schedule_forgets_deleted_disabled_and_repointed_feeds() { + let t0 = Instant::now(); + let kept = schedule_feed("kept", "https://example.test/kept", 900); + let deleted = schedule_feed("deleted", "https://example.test/deleted", 900); + let mut toggled = schedule_feed("toggled", "https://example.test/toggled", 900); + let mut moved = schedule_feed("moved", "https://example.test/old", 900); + let mut schedule = FeedSchedule::default(); + for feed in [&kept, &deleted, &toggled, &moved] { + schedule.mark_checked(feed, t0); + } + + toggled.enabled = false; + moved.url = "https://example.test/new".into(); + let feeds = vec![kept.clone(), toggled.clone(), moved.clone()]; + schedule.sync(&feeds); + + assert!(!schedule.is_due(&kept, t0)); + assert!(!schedule.last_checked.contains_key("deleted")); + assert!(!schedule.last_checked.contains_key("toggled")); + assert!( + schedule.is_due(&moved, t0), + "a re-pointed feed is checked at once" + ); + assert_eq!( + schedule.next_due(&[toggled.clone()]), + None, + "disabled feeds never wake the loop" + ); + + toggled.enabled = true; + schedule.sync(std::slice::from_ref(&toggled)); + assert!( + schedule.is_due(&toggled, t0), + "a re-enabled feed is checked at once" + ); + } + + #[test] + fn schedule_clamps_intervals_below_the_floor_and_warns_once_per_feed() { + let t0 = Instant::now(); + let mut zero = schedule_feed("zero", "https://example.test/zero", 0); + let tiny = schedule_feed("tiny", "https://example.test/tiny", 5); + let ok = schedule_feed("ok", "https://example.test/ok", MIN_POLL_INTERVAL_SECS); + let mut schedule = FeedSchedule::default(); + + let feeds = vec![zero.clone(), tiny.clone(), ok.clone()]; + let mut warned = schedule.sync(&feeds); + warned.sort(); + assert_eq!(warned, vec!["tiny".to_string(), "zero".to_string()]); + assert!(schedule.sync(&feeds).is_empty(), "warns only once per feed"); + + let floor = Duration::from_secs(MIN_POLL_INTERVAL_SECS); + for feed in [&zero, &tiny, &ok] { + schedule.mark_checked(feed, t0); + assert_eq!(schedule.due_at(feed), Some(t0 + floor), "{}", feed.name); + assert!(!schedule.is_due(feed, t0), "{} must not spin", feed.name); + } + + // Fixing the interval and then breaking it again warns afresh. + zero.poll_interval_secs = 900; + assert!(schedule.sync(std::slice::from_ref(&zero)).is_empty()); + zero.poll_interval_secs = 0; + assert_eq!( + schedule.sync(std::slice::from_ref(&zero)), + vec!["zero".to_string()] + ); + } } diff --git a/crates/nzb-web/src/sabnzbd_compat.rs b/crates/nzb-web/src/sabnzbd_compat.rs index 13b2562e..825140de 100644 --- a/crates/nzb-web/src/sabnzbd_compat.rs +++ b/crates/nzb-web/src/sabnzbd_compat.rs @@ -16,7 +16,7 @@ use crate::nzb_core::models::*; use crate::nzb_core::nzb_parser; use crate::error::ApiError; -use crate::queue_manager::job_id_matches; +use crate::queue_manager::{HistoryRetryOutcome, job_id_matches}; use crate::state::AppState; /// SABnzbd release whose public response contract this compatibility layer @@ -55,6 +55,11 @@ pub struct SabApiRequest { /// SABnzbd's `script` parameter is accepted and ignored (unknown query /// fields are not rejected); RustNZB has no per-job scripts. pub pp: Option, + /// Sort field for `mode=queue&name=sort` (`avg_age`, `name`, `size`, + /// `remaining`). + pub sort: Option, + /// Sort direction for `mode=queue&name=sort` (`asc` or `desc`). + pub dir: Option, } /// Validate API key. Returns Err with JSON response on failure. @@ -258,13 +263,19 @@ async fn handle_addurl( // Content-Disposition, then the URL path), but when neither override nor // disposition names the job the decompressed member is used, so a fetched // .nzb.gz is named after the inner NZB rather than the archive. - let job_name = if nzbname.as_deref().and_then(clean_nzb_name).is_none() + // Split SABnzbd's inline password off `nzbname` before it is cleaned. + let (nzbname, nzbname_pw) = take_inline_password_opt(nzbname.as_deref()); + let (job_name, name_pw) = if nzbname.as_deref().and_then(clean_nzb_name).is_none() && content_disposition .as_deref() .and_then(content_disposition_filename) .is_none() { - clean_nzb_name(&nzb_file_name).unwrap_or_else(|| "unknown".to_string()) + let (member, member_pw) = take_inline_password(&nzb_entry_file_name(&nzb_file_name)); + ( + clean_nzb_name(&member).unwrap_or_else(|| "unknown".to_string()), + member_pw, + ) } else { addurl_job_name( nzbname.as_deref(), @@ -272,6 +283,7 @@ async fn handle_addurl( &fetch_plan.url, ) }; + let inline_password = nzbname_pw.or(name_pw); match nzb_parser::parse_nzb(&job_name, &data) { Ok(mut job) => { @@ -284,11 +296,9 @@ async fn handle_addurl( apply_sab_add_priority(&mut job, p); } - // API-provided password overrides NZB metadata password - if let Some(ref pw) = password { - job.password = Some(pw.clone()); - } + apply_job_password(&mut job, password, inline_password); job.pp_override = sab_pp_override(pp.as_deref()); + job.delete_archives = sab_pp_delete_archives(pp.as_deref()); let qm = &state.queue_manager; job.work_dir = qm.incomplete_dir().join(&job.id); @@ -352,23 +362,55 @@ fn addurl_job_name( nzbname: Option<&str>, content_disposition: Option<&str>, url: &reqwest::Url, -) -> String { - nzbname - .and_then(clean_nzb_name) - .or_else(|| { - content_disposition - .and_then(content_disposition_filename) - .as_deref() - .and_then(clean_nzb_name) - }) +) -> (String, Option) { + // The fetched file's name: Content-Disposition, else the URL path. Like + // SABnzbd, its inline password applies even when `nzbname` names the job. + let file_name = content_disposition + .and_then(content_disposition_filename) + .filter(|name| clean_nzb_name(name).is_some()) .or_else(|| { url.path_segments() .and_then(|mut segments| segments.next_back()) .map(percent_decode_lossy) - .as_deref() - .and_then(clean_nzb_name) }) - .unwrap_or_else(|| "unknown".to_string()) + .map(|name| take_inline_password(&nzb_entry_file_name(&name))); + let file_pw = file_name.as_ref().and_then(|(_, pw)| pw.clone()); + let name = nzbname + .and_then(clean_nzb_name) + .or_else(|| { + file_name + .as_ref() + .and_then(|(name, _)| clean_nzb_name(name)) + }) + .unwrap_or_else(|| "unknown".to_string()); + (name, file_pw) +} + +/// Split SABnzbd's inline job password (`name{{pw}}`, `name/pw`) off a +/// client-supplied name. Returns the name to use and the password, if any; +/// an empty inline password (`name{{}}`) is treated as none. +fn take_inline_password(raw: &str) -> (String, Option) { + match crate::nzb_core::path::split_job_password(raw) { + Some((name, pw)) => (name, Some(pw).filter(|pw| !pw.is_empty())), + None => (raw.to_string(), None), + } +} + +fn take_inline_password_opt(raw: Option<&str>) -> (Option, Option) { + match raw.map(take_inline_password) { + Some((name, pw)) => (Some(name), pw), + None => (None, None), + } +} + +/// Set the job's archive password: an explicit `password` parameter wins +/// over an inline one from the job name, which wins over the NZB's +/// `` (already on the job). Empty values count as +/// unset, as in SABnzbd. +fn apply_job_password(job: &mut NzbJob, explicit: Option, inline: Option) { + if let Some(pw) = explicit.filter(|pw| !pw.is_empty()).or(inline) { + job.password = Some(pw); + } } /// Reduce a client- or server-supplied NZB name to a bare job name: last @@ -569,11 +611,7 @@ async fn read_multipart_fields( nzbname: &mut Option, pp: &mut Option, ) -> Result<(), ApiError> { - while let Some(field) = multipart - .next_field() - .await - .map_err(|e| ApiError::from(anyhow::anyhow!("Multipart error: {e}")))? - { + while let Some(field) = multipart.next_field().await.map_err(ApiError::from)? { let field_name = field.name().unwrap_or("").to_string(); match field_name.as_str() { "mode" => { @@ -607,10 +645,7 @@ async fn read_multipart_fields( .file_name() .map(|s| s.to_string()) .unwrap_or_else(|| "unknown.nzb".into()); - let data = field - .bytes() - .await - .map_err(|e| ApiError::from(anyhow::anyhow!("Read error: {e}")))?; + let data = field.bytes().await.map_err(ApiError::from)?; *nzb_data = Some((file_name, data.to_vec())); } else if let Ok(text) = field.text().await { *name = Some(text); @@ -621,10 +656,7 @@ async fn read_multipart_fields( .file_name() .map(|s| s.to_string()) .unwrap_or_else(|| "unknown.nzb".into()); - let data = field - .bytes() - .await - .map_err(|e| ApiError::from(anyhow::anyhow!("Read error: {e}")))?; + let data = field.bytes().await.map_err(ApiError::from)?; *nzb_data = Some((file_name, data.to_vec())); } "value" | "url" => { @@ -711,11 +743,19 @@ async fn dispatch_post( let (entry_name, data) = nzbs.pop().expect("extract_nzbs returns at least one NZB"); let file_name = nzb_entry_file_name(&entry_name); - let job_name = query_req - .nzbname + // SABnzbd's `name{{password}}` / `name/password` convention: + // split it off every name source before naming (and therefore + // sanitizing) the job. `nzbname` must be split before + // `clean_nzb_name`, which would keep only the part after `/`. + let (nzbname, nzbname_pw) = take_inline_password_opt(query_req.nzbname.as_deref()); + let (name, name_pw) = take_inline_password_opt(name.as_deref()); + let (file_name, file_pw) = take_inline_password(&file_name); + let inline_password = nzbname_pw.or(name_pw).or(file_pw); + + let job_name = nzbname .as_deref() .and_then(clean_nzb_name) - .or_else(|| name.clone()) + .or(name) .unwrap_or_else(|| { file_name .strip_suffix(".nzb") @@ -734,11 +774,9 @@ async fn dispatch_post( apply_sab_add_priority(&mut job, p); } - // API-provided password overrides NZB metadata password - if let Some(ref pw) = password { - job.password = Some(pw.clone()); - } + apply_job_password(&mut job, password, inline_password); job.pp_override = sab_pp_override(query_req.pp.as_deref()); + job.delete_archives = sab_pp_delete_archives(query_req.pp.as_deref()); let qm = &state.queue_manager; job.work_dir = qm.incomplete_dir().join(&job.id); @@ -840,6 +878,8 @@ async fn dispatch_post( del_files: query_req.del_files, nzbname: query_req.nzbname, pp: query_req.pp, + sort: query_req.sort, + dir: query_req.dir, }; Ok(dispatch_mode( state, @@ -964,13 +1004,14 @@ fn dispatch_mode(state: &AppState, mode: &str, req: &SabApiRequest) -> Json handle_retry(state, req), + "retry_all" => handle_retry_all(state), + // SABnzbd `_api_warnings`. RustNZB does not keep a SABnzbd-style - // warnings list (queue/fullstatus report `have_warnings: "0"` and - // `warnings: []`), so this is the same empty list; `name=clear` - // is accepted. + // warnings log; the only warning reported is the live disk-space + // hold (see `sab_warnings`). `name=clear` is accepted. "warnings" => match req.name.as_deref() { Some("clear") => Json(serde_json::json!({ "status": true })), - _ => Json(serde_json::json!({ "warnings": Vec::::new() })), + _ => Json(serde_json::json!({ "warnings": sab_warnings(&state.queue_manager) })), }, _ => Json(serde_json::json!({ @@ -1019,6 +1060,7 @@ fn handle_fullstatus(state: &AppState) -> Json { let pause_int = qm.pause_remaining_secs().unwrap_or(0).max(0).to_string(); let (free1, total1, free1_norm) = sab_disk_space(&qm.incomplete_dir()); let (free2, total2, free2_norm) = sab_disk_space(&qm.complete_dir()); + let warnings = sab_warnings(qm); Json(serde_json::json!({ "status": { @@ -1045,7 +1087,7 @@ fn handle_fullstatus(state: &AppState) -> Json { "finishaction": serde_json::Value::Null, "folders": Vec::::new(), "have_quota": false, - "have_warnings": "0", + "have_warnings": warnings.len().to_string(), "internetbandwidth": 0, "ipv6": serde_json::Value::Null, "left_quota": "0 B", @@ -1077,7 +1119,7 @@ fn handle_fullstatus(state: &AppState) -> Json { "uptime": format_sab_age(state.started_at.elapsed().as_secs()), "url_base": "", "version": SABNZBD_COMPAT_VERSION, - "warnings": Vec::::new(), + "warnings": warnings, "webdir": "", "weblogfile": serde_json::Value::Null, "windows": cfg!(target_os = "windows"), @@ -1138,13 +1180,42 @@ fn handle_server_stats(state: &AppState) -> Json { // Mode handlers // --------------------------------------------------------------------------- +/// `mode=queue&name=sort&sort=&dir=`, as SABnzbd's +/// `_api_queue_sort`. Only `dir=desc` reverses, like SABnzbd. Without +/// `sort`, the earlier RustNZB form (remaining percentage, direction in +/// `value`) still works. +fn handle_queue_sort(state: &AppState, req: &SabApiRequest) -> Json { + let is_desc = |value: &str| { + value.eq_ignore_ascii_case("desc") || value.eq_ignore_ascii_case("descending") + }; + let field = match req.sort.as_deref().filter(|sort| !sort.is_empty()) { + None => crate::QueueSortField::Remaining, + Some(sort) => match sort.parse() { + Ok(field) => field, + Err(()) => { + return Json(serde_json::json!({ + "status": false, + "error": format!("Unknown sort field: {sort}") + })); + } + }, + }; + let descending = req + .dir + .as_deref() + .or(req.value.as_deref()) + .is_some_and(is_desc); + state.queue_manager.sort_queue(field, !descending); + Json(serde_json::json!({ "status": true })) +} + fn handle_queue(state: &AppState, req: &SabApiRequest) -> Json { let qm = &state.queue_manager; // Sub-commands dispatched via mode=queue&name=, matching SABnzbd's // real `_api_queue_table` (delete, pause, resume, priority, rename, - // purge, change_complete_action). `sort` and `delete_nzf` have no - // equivalent capability in RustNZB's queue manager yet. + // purge, change_complete_action, sort). `delete_nzf` has no equivalent + // capability in RustNZB's queue manager yet. match req.name.as_deref() { Some("delete") => return handle_queue_delete(state, req), Some("pause") => return handle_queue_item_pause(state, req), @@ -1152,16 +1223,7 @@ fn handle_queue(state: &AppState, req: &SabApiRequest) -> Json return handle_queue_priority(state, req), Some("rename") => return handle_queue_rename(state, req), Some("purge") => return handle_queue_purge(state), - Some("sort") => { - let ascending = !matches!( - req.value.as_deref(), - Some(value) - if value.eq_ignore_ascii_case("descending") - || value.eq_ignore_ascii_case("desc") - ); - qm.sort_by_remaining_percentage(ascending); - return Json(serde_json::json!({ "status": true })); - } + Some("sort") => return handle_queue_sort(state, req), Some("change_complete_action") => return Json(serde_json::json!({ "status": true })), _ => {} } @@ -1171,7 +1233,10 @@ fn handle_queue(state: &AppState, req: &SabApiRequest) -> Json Json Vec { + qm.disk_space_hold_since() + .filter(|_| qm.is_paused()) + .map(|since| { + serde_json::json!({ + "type": "WARNING", + "text": "Too little diskspace forcing PAUSE", + "time": since.timestamp(), + }) + }) + .into_iter() + .collect() +} + /// SABnzbd's `speedlimit` is a percentage of the configured maximum /// bandwidth. RustNZB has no maximum-bandwidth setting, so an active limit /// is reported as 100% and no limit as "0"; the absolute value is carried @@ -1255,6 +1340,7 @@ fn build_queue_response( speed_bps: u64, speed_limit_bps: u64, req: &SabApiRequest, + pp_level: impl Fn(&NzbJob) -> u8, ) -> serde_json::Value { let start = req.start.unwrap_or(0); let limit = req.limit.unwrap_or(0); @@ -1315,7 +1401,14 @@ fn build_queue_response( if queue_totals_include(job) { running_bytes = running_bytes.saturating_add(remaining_bytes(job)); } - SabQueueSlot::from_job(job, start + offset, paused, running_bytes, speed_bps) + SabQueueSlot::from_job( + job, + start + offset, + paused, + running_bytes, + speed_bps, + pp_level(job), + ) }) .collect(); @@ -1588,6 +1681,10 @@ fn handle_history(state: &AppState, req: &SabApiRequest) -> Json Json serde_json::Value { let mut slots: Vec = postprocessing .iter() - .map(SabHistorySlot::from_postprocessing) + .map(|(job, level)| SabHistorySlot::from_postprocessing(job, *level)) .chain(entries.iter().map(SabHistorySlot::from_entry)) .filter(|slot| history_slot_matches(slot, req)) .collect(); @@ -2020,61 +2117,63 @@ fn handle_retry(state: &AppState, req: &SabApiRequest) -> Json data, - Ok(None) => { - return Json(serde_json::json!({ - "status": false, - "error": "The original NZB data is unavailable for this history job" - })); - } - Err(error) => { - return Json(serde_json::json!({ "status": false, "error": error.to_string() })); - } - }; + // At most one retry of an entry is in flight: a repeat while it is still + // queued or running returns the existing job's nzo_id (status true) + // rather than enqueueing a duplicate. + match retry_history_job(state, &entry) { + Ok(nzo_id) => Json(serde_json::json!({ "status": true, "nzo_ids": [nzo_id] })), + Err(error) => Json(serde_json::json!({ "status": false, "error": error })), + } +} - let retry_data = match state.queue_manager.history_get_retry_data(&entry.id) { - Ok(data) => data, - Err(error) => { - return Json(serde_json::json!({ "status": false, "error": error.to_string() })); - } - }; - let job = match state +/// SABnzbd `_api_retry_all`: retry every history entry that history reports +/// as retryable (`retry: true`). SABnzbd 5.0.4 answers with +/// `{"status": [nzo_id, ...]}`, one element per attempted entry and `null` +/// where that retry failed, so this does the same. +fn handle_retry_all(state: &AppState) -> Json { + let entries = state .queue_manager - .prepare_retry_job(&entry, &data, retry_data.as_deref()) - { - Ok(job) => job, - Err(error) => { - return Json(serde_json::json!({ - "status": false, - "error": format!("Failed to parse stored NZB: {error}") - })); - } - }; + .history_list(i64::MAX as usize) + .unwrap_or_default(); + let nzo_ids: Vec> = entries + .iter() + .filter(|entry| entry.status == JobStatus::Failed && entry.nzb_data.is_some()) + .map(|entry| { + retry_history_job(state, entry) + .inspect_err(|error| { + tracing::warn!(history_id = %entry.id, %error, "retry_all: retry failed"); + }) + .ok() + }) + .collect(); + Json(serde_json::json!({ "status": nzo_ids })) +} - let nzo_id = format!("SABnzbd_nzo_{}", sab_id_prefix(&job.id)); - if let Err(error) = state.queue_manager.add_job(job, Some(data)) { - return Json(serde_json::json!({ "status": false, "error": error.to_string() })); +/// Re-queue one history entry from its stored NZB, returning the new SAB +/// `nzo_id`. Shared by `mode=retry` and `mode=retry_all`. A repeat while the +/// earlier retry is still queued or running returns that job's nzo_id instead +/// of enqueueing a duplicate. +fn retry_history_job(state: &AppState, entry: &HistoryEntry) -> Result { + match state.queue_manager.retry_history_entry(entry) { + Ok(HistoryRetryOutcome::Started(job_id) | HistoryRetryOutcome::InProgress(job_id)) => { + Ok(format!("SABnzbd_nzo_{}", sab_id_prefix(&job_id))) + } + Ok(HistoryRetryOutcome::NoNzbData) => { + Err("The original NZB data is unavailable for this history job".into()) + } + Err(error) => Err(error.to_string()), } - - tracing::info!(history_id = %entry.id, retried_id = %nzo_id, "History job retried via arr API"); - Json(serde_json::json!({ "status": true, "nzo_ids": [nzo_id] })) } fn handle_switch(state: &AppState, req: &SabApiRequest) -> Json { let target = req.value.as_deref().unwrap_or(""); - let position = req - .value2 - .as_deref() - .or(req.name.as_deref()) - .and_then(|value| value.parse::().ok()); - - let Some(position) = position else { + let destination = req.value2.as_deref().or(req.name.as_deref()).unwrap_or(""); + if destination.is_empty() { return Json(serde_json::json!({ "status": false, "error": "Missing or invalid target queue position" })); - }; + } if target.is_empty() { return Json(serde_json::json!({ "status": false, "error": "No job ID" })); } @@ -2082,15 +2181,51 @@ fn handle_switch(state: &AppState, req: &SabApiRequest) -> Json` id, so resolve // it to the full job id by prefix -- as every other per-job handler does // -- before handing it to `move_job`, which matches ids exactly. - let search_id = target.strip_prefix("SABnzbd_nzo_").unwrap_or(target); let qm = &state.queue_manager; - let Some(job) = qm - .get_jobs() - .into_iter() - .find(|job| job_id_matches(&job.id, search_id)) - else { + let find_job = |requested: &str| { + let search_id = requested.strip_prefix("SABnzbd_nzo_").unwrap_or(requested); + qm.get_jobs() + .into_iter() + .find(|job| job_id_matches(&job.id, search_id)) + }; + let Some(mut job) = find_job(target) else { return Json(serde_json::json!({ "status": false, "error": "Job not found" })); }; + + // `value2` is either a queue index or, as in SABnzbd's + // `NzbQueue.switch`, another job's nzo_id: the job then takes that job's + // place in the queue and adopts its priority. + let position = match destination.parse::() { + Ok(position) => position, + Err(_) => { + let Some(other) = find_job(destination) else { + return Json(serde_json::json!({ + "status": false, + "error": "Missing or invalid target queue position" + })); + }; + if other.priority != job.priority { + if let Err(error) = qm.set_job_priority(&job.id, other.priority) { + return Json( + serde_json::json!({ "status": false, "error": error.to_string() }), + ); + } + job.priority = other.priority; + } + // Look the target up after any priority reorder. + match qm + .get_jobs() + .iter() + .position(|queued| queued.id == other.id) + { + Some(position) => position, + None => { + return Json(serde_json::json!({ "status": false, "error": "Job not found" })); + } + } + } + }; + match qm.move_job(&job.id, position) { Ok(()) => { let new_position = qm @@ -2376,6 +2511,27 @@ fn sab_pp_override(pp: Option<&str>) -> Option { } } +/// The archive-deletion flag for SABnzbd's `pp`: 2 (+repair/unpack) keeps +/// the archives, 3 (+delete) removes them. Other values leave the default. +fn sab_pp_delete_archives(pp: Option<&str>) -> Option { + match pp?.trim().parse::().ok()? { + 2 => Some(false), + 3 => Some(true), + _ => None, + } +} + +/// The SABnzbd-scale level reported for a RustNZB level: an unpacking job +/// that keeps its archives is SABnzbd's 2 (+repair/unpack without +delete), +/// so `pp=2` reads back as `unpackopts` "2" and history `pp` "U". +fn sab_reported_level(level: u8, delete_archives: Option) -> u8 { + if level >= 2 && delete_archives == Some(false) { + 2 + } else { + level + } +} + /// Convert arr-protocol priority string to our Priority enum. fn sab_priority_to_priority(s: &str) -> Priority { match s.trim() { @@ -2425,6 +2581,7 @@ impl SabQueueSlot { globally_paused: bool, running_bytes: u64, speed_bps: u64, + pp_level: u8, ) -> Self { let mb = job.total_bytes as f64 / 1_048_576.0; let mbleft = remaining_bytes(job) as f64 / 1_048_576.0; @@ -2438,7 +2595,7 @@ impl SabQueueSlot { Self { index, nzo_id: queue_nzo_id(job), - unpackopts: "3".into(), + unpackopts: sab_unpackopts(sab_reported_level(pp_level, job.delete_archives)).into(), script: "None".into(), filename: job.name.clone(), labels: Vec::new(), @@ -2510,6 +2667,32 @@ fn sab_queue_status(status: JobStatus) -> &'static str { } } +/// SABnzbd's queue `unpackopts` for an effective post-processing level: the +/// level as a string, on the scale `pp` is accepted on (see +/// [`sab_pp_override`]) and history reports (see [`sab_pp_label`]), so a job +/// re-added with its reported `unpackopts` keeps its level. Out-of-range +/// levels fall back to the default, 3. +fn sab_unpackopts(level: u8) -> &'static str { + match level { + 0 => "0", + 1 => "1", + 2 => "2", + _ => "3", + } +} + +/// SABnzbd's history `pp` label for an effective post-processing level +/// (`sabnzbd/constants.py::PP_LOOKUP`); unknown levels map to `"X"` as there. +fn sab_pp_label(level: u8) -> &'static str { + match level { + 0 => "", + 1 => "R", + 2 => "U", + 3 => "D", + _ => "X", + } +} + #[derive(Serialize)] struct SabHistorySlot { completed: i64, @@ -2572,7 +2755,13 @@ impl SabHistorySlot { name: entry.name.clone(), nzb_name: format!("{}.nzb", entry.name), category: sab_category_label(&entry.category), - pp: "D".into(), + // Rows written before the level was recorded keep reporting "D". + pp: entry + .post_processing + .map_or("D", |level| { + sab_pp_label(sab_reported_level(level, entry.delete_archives)) + }) + .into(), script: String::new(), report: String::new(), url: String::new(), @@ -2619,7 +2808,7 @@ impl SabHistorySlot { } } - fn from_postprocessing(job: &NzbJob) -> Self { + fn from_postprocessing(job: &NzbJob, pp_level: u8) -> Self { let path = job.work_dir.to_string_lossy().to_string(); Self { completed: job @@ -2629,7 +2818,7 @@ impl SabHistorySlot { name: job.name.clone(), nzb_name: format!("{}.nzb", job.name), category: sab_category_label(&job.category), - pp: "D".into(), + pp: sab_pp_label(sab_reported_level(pp_level, job.delete_archives)).into(), script: String::new(), report: String::new(), url: String::new(), @@ -2835,6 +3024,7 @@ mod tests { error_message: None, speed_bps: 0, pp_override: None, + delete_archives: None, server_stats: Vec::new(), files: Vec::new(), } @@ -2867,6 +3057,8 @@ mod tests { }], error_message: (status == JobStatus::Failed).then(|| "broken archive".into()), failure_code: (status == JobStatus::Failed).then_some(JobFailureCode::ArchiveInvalid), + post_processing: None, + delete_archives: None, server_stats: Vec::new(), nzb_data: (status == JobStatus::Failed).then(Vec::new), retry_data: None, @@ -2896,6 +3088,7 @@ mod tests { error_message: None, speed_bps: 0, pp_override: None, + delete_archives: None, server_stats: Vec::new(), files: Vec::new(), } @@ -2934,6 +3127,7 @@ mod tests { 1_048_576, 2_097_152, &SabApiRequest::default(), + |_| 3, ); let queue = &response["queue"]; let slot = &queue["slots"][0]; @@ -2948,7 +3142,7 @@ mod tests { assert_eq!(slot["index"], 0); assert_eq!(slot["nzo_id"], "SABnzbd_nzo_1234567890ab"); - assert_eq!(slot["unpackopts"], "3"); + assert!(slot["unpackopts"].is_string()); assert_eq!(slot["script"], "None"); assert_eq!(slot["labels"], serde_json::json!([])); assert_eq!(slot["password"], "secret"); @@ -2965,6 +3159,7 @@ mod tests { 0, 0, &SabApiRequest::default(), + |_| 3, ); assert_eq!(response["queue"]["status"], "Idle"); assert_eq!(response["queue"]["timeleft"], "0:00:00"); @@ -2985,6 +3180,7 @@ mod tests { 0, 0, &SabApiRequest::default(), + |_| 3, ); assert_eq!(response["queue"]["status"], "Downloading"); @@ -2999,13 +3195,14 @@ mod tests { 0, 0, &SabApiRequest::default(), + |_| 3, ); assert_eq!(paused["queue"]["status"], "Paused"); } #[test] fn empty_and_paused_queues_have_sab_statuses() { - let empty = build_queue_response(&[], false, 0, 0, &SabApiRequest::default()); + let empty = build_queue_response(&[], false, 0, 0, &SabApiRequest::default(), |_| 3); assert_eq!(empty["queue"]["status"], "Idle"); assert_eq!(empty["queue"]["slots"], serde_json::json!([])); assert_eq!(empty["queue"]["noofslots_total"], 0); @@ -3016,6 +3213,7 @@ mod tests { 1_048_576, 0, &SabApiRequest::default(), + |_| 3, ); assert_eq!(paused["queue"]["status"], "Paused"); assert_eq!(paused["queue"]["slots"][0]["timeleft"], "0:00:00"); @@ -3068,7 +3266,7 @@ mod tests { limit: Some(1), ..SabApiRequest::default() }; - let response = build_queue_response(&jobs, false, 0, 0, &req); + let response = build_queue_response(&jobs, false, 0, 0, &req, |_| 3); let queue = &response["queue"]; assert_eq!(queue["noofslots_total"], 4); @@ -3092,7 +3290,7 @@ mod tests { nzo_ids: Some("SABnzbd_nzo_high-priorit".into()), ..SabApiRequest::default() }; - let response = build_queue_response(&[high, normal], false, 0, 0, &req); + let response = build_queue_response(&[high, normal], false, 0, 0, &req, |_| 3); assert_eq!(response["queue"]["noofslots"], 1); assert_eq!(response["queue"]["slots"][0]["filename"], "First"); @@ -3115,6 +3313,8 @@ mod tests { stages: Vec::new(), error_message: None, failure_code: None, + post_processing: None, + delete_archives: None, server_stats: Vec::new(), nzb_data: None, retry_data: None, @@ -3189,6 +3389,122 @@ mod tests { assert!(slots[0]["completeness"].is_null()); } + /// SABnzbd's queue `unpackopts` is the job's post-processing level as a + /// string, on the same scale `pp` is accepted on and history reports. + #[test] + fn queue_slot_unpackopts_reflects_the_effective_level() { + for level in 0..=3_u8 { + let jobs = vec![queue_job("upk", "Upk", "tv", JobStatus::Queued)]; + let response = + build_queue_response(&jobs, false, 0, 0, &SabApiRequest::default(), |_| level); + assert_eq!( + response["queue"]["slots"][0]["unpackopts"], + level.to_string(), + "level {level}" + ); + } + } + + /// A job added with SABnzbd `pp=2` (repair+unpack, keep the archives) + /// reports `unpackopts` "2" in the queue and `pp` "U" in history, so a + /// client re-adding it with what it reads back gets the same behaviour. + #[tokio::test] + async fn addfile_pp2_reports_unpack_without_delete() { + let job = addfile_multipart(SabApiRequest::default(), &[("pp", "2")]).await; + assert_eq!(job.delete_archives, Some(false)); + let level = job.pp_override.expect("pp override"); + + let queue = build_queue_response( + std::slice::from_ref(&job), + false, + 0, + 0, + &SabApiRequest::default(), + |_| level, + ); + assert_eq!(queue["queue"]["slots"][0]["unpackopts"], "2"); + + let running = + build_history_response(&[], &[(job.clone(), level)], &SabApiRequest::default(), 1); + assert_eq!(running["history"]["slots"][0]["pp"], "U"); + + let mut entry = history_entry("keep", "Keep", "movies", JobStatus::Completed, 1); + entry.post_processing = Some(level); + entry.delete_archives = job.delete_archives; + assert_eq!(SabHistorySlot::from_entry(&entry).pp, "U"); + // The default (delete) still reports "D". + entry.delete_archives = Some(true); + assert_eq!(SabHistorySlot::from_entry(&entry).pp, "D"); + } + + /// The live queue resolves the level from the job's `pp` override, else + /// its category. + #[tokio::test] + async fn queue_unpackopts_uses_job_override_then_category() { + let test_state = test_state(); + test_state + .state + .queue_manager + .set_categories(vec![CategoryConfig { + name: "unpackonly".into(), + post_processing: 2, + ..CategoryConfig::default() + }]); + add_live_job_with_category(&test_state, "upk-category", "unpackonly"); + let mut job = queue_job("upk-override", "Override", "unpackonly", JobStatus::Queued); + job.pp_override = Some(0); + let general = &test_state.state.config().general; + job.work_dir = general.incomplete_dir.join("upk-override"); + job.output_dir = general.complete_dir.join("upk-override"); + test_state + .state + .queue_manager + .add_job(job, None) + .expect("add override job"); + + let queue = dispatch_mode(&test_state.state, "queue", &SabApiRequest::default()).0; + let slots = queue["queue"]["slots"].as_array().expect("slots"); + let unpackopts = |nzo_id: &str| { + slots + .iter() + .find(|slot| { + slot["nzo_id"] + .as_str() + .is_some_and(|id| id.starts_with(nzo_id)) + }) + .map(|slot| slot["unpackopts"].clone()) + .expect("slot present") + }; + assert_eq!(unpackopts("SABnzbd_nzo_upk-category"), "2"); + assert_eq!(unpackopts("SABnzbd_nzo_upk-override"), "0"); + } + + /// SABnzbd reports the job's post-processing level as + /// `PP_LOOKUP = {0: "", 1: "R", 2: "U", 3: "D"}`. + #[test] + fn history_slot_pp_reflects_the_effective_level() { + for (level, label) in [(0, ""), (1, "R"), (2, "U"), (3, "D")] { + let mut entry = history_entry("pp", "PP", "movies", JobStatus::Completed, 1); + entry.post_processing = Some(level); + assert_eq!( + SabHistorySlot::from_entry(&entry).pp, + label, + "level {level}" + ); + } + // Rows recorded before the level was stored keep the old answer. + let legacy = history_entry("legacy", "Legacy", "movies", JobStatus::Completed, 1); + assert_eq!(SabHistorySlot::from_entry(&legacy).pp, "D"); + + let response = build_history_response( + &[], + &[(postprocessing_job(), 1)], + &SabApiRequest::default(), + 1, + ); + assert_eq!(response["history"]["slots"][0]["pp"], "R"); + } + #[test] fn history_includes_postprocessing_before_terminal_slots() { let response = build_history_response( @@ -3199,7 +3515,7 @@ mod tests { JobStatus::Completed, 1, )], - &[postprocessing_job()], + &[(postprocessing_job(), 3)], &SabApiRequest::default(), 4, ); @@ -3321,6 +3637,7 @@ mod tests { error_message: None, speed_bps: 0, pp_override: None, + delete_archives: None, server_stats: Vec::new(), files: Vec::new(), }; @@ -3364,6 +3681,8 @@ mod tests { stages: Vec::new(), error_message: None, failure_code: None, + post_processing: None, + delete_archives: None, server_stats: Vec::new(), nzb_data: None, retry_data: None, @@ -3450,6 +3769,7 @@ mod tests { error_message: None, speed_bps: 0, pp_override: None, + delete_archives: None, server_stats: Vec::new(), files: Vec::new(), }; @@ -3482,6 +3802,7 @@ mod tests { error_message: None, speed_bps: 0, pp_override: None, + delete_archives: None, server_stats: Vec::new(), files: Vec::new(), }; @@ -3564,6 +3885,119 @@ mod tests { ); } + fn queue_order(test_state: &TestState) -> Vec { + test_state + .state + .queue_manager + .get_jobs() + .into_iter() + .map(|job| job.id) + .collect() + } + + /// SABnzbd's `switch` also accepts another job's nzo_id as `value2`, + /// moving `value` into that job's queue position (`NzbQueue.switch`). + #[tokio::test] + async fn switch_accepts_target_nzo_id_as_value2() { + let test_state = test_state(); + let a = "aaaaaaaa-1111-0000-0000-000000000001"; + let b = "bbbbbbbb-2222-0000-0000-000000000002"; + let c = "cccccccc-3333-0000-0000-000000000003"; + for id in [a, b, c] { + add_live_job(&test_state, id); + } + + let req = SabApiRequest { + mode: Some("switch".into()), + value: Some("SABnzbd_nzo_cccccccc-333".into()), + value2: Some("SABnzbd_nzo_aaaaaaaa-111".into()), + ..SabApiRequest::default() + }; + let response = dispatch_mode(&test_state.state, "switch", &req).0; + assert_eq!(response["status"], serde_json::json!(true), "{response:?}"); + assert_eq!(response["result"]["position"], serde_json::json!(0)); + assert_eq!(response["result"]["priority"], serde_json::json!(0)); + assert_eq!(queue_order(&test_state), vec![c, a, b]); + + // Moving down: the moved job lands in the target's old slot. + let req = SabApiRequest { + mode: Some("switch".into()), + value: Some(format!("SABnzbd_nzo_{}", &c[..12])), + value2: Some(b.into()), + ..SabApiRequest::default() + }; + let response = dispatch_mode(&test_state.state, "switch", &req).0; + assert_eq!(response["status"], serde_json::json!(true), "{response:?}"); + assert_eq!(response["result"]["position"], serde_json::json!(2)); + assert_eq!(queue_order(&test_state), vec![a, b, c]); + } + + /// Like SABnzbd, a job switched next to a job of another priority adopts + /// the target's priority, and the response reports it. + #[tokio::test] + async fn switch_to_nzo_id_adopts_target_priority() { + let test_state = test_state(); + let a = "aaaaaaaa-1111-0000-0000-000000000001"; + let b = "bbbbbbbb-2222-0000-0000-000000000002"; + add_live_job(&test_state, a); + add_live_job(&test_state, b); + test_state + .state + .queue_manager + .set_job_priority(a, Priority::High) + .expect("raise priority"); + + let req = SabApiRequest { + mode: Some("switch".into()), + value: Some(b.into()), + value2: Some(a.into()), + ..SabApiRequest::default() + }; + let response = dispatch_mode(&test_state.state, "switch", &req).0; + assert_eq!(response["status"], serde_json::json!(true), "{response:?}"); + assert_eq!(response["result"]["position"], serde_json::json!(0)); + assert_eq!( + response["result"]["priority"], + serde_json::json!(sab_priority_code(Priority::High)) + ); + assert_eq!(queue_order(&test_state), vec![b, a]); + let moved = test_state + .state + .queue_manager + .get_jobs() + .into_iter() + .find(|job| job.id == b) + .expect("job b queued"); + assert_eq!(moved.priority, Priority::High); + } + + /// A `value2` that is neither a number nor a known job id fails without + /// touching the queue. + #[tokio::test] + async fn switch_rejects_unknown_target_nzo_id() { + let test_state = test_state(); + let a = "aaaaaaaa-1111-0000-0000-000000000001"; + let b = "bbbbbbbb-2222-0000-0000-000000000002"; + add_live_job(&test_state, a); + add_live_job(&test_state, b); + + for value2 in ["SABnzbd_nzo_ffffffff-999", "SABnzbd_nzo_", "bbbb", "x"] { + let req = SabApiRequest { + mode: Some("switch".into()), + value: Some(b.into()), + value2: Some(value2.into()), + ..SabApiRequest::default() + }; + let response = dispatch_mode(&test_state.state, "switch", &req).0; + assert_eq!( + response["status"], + serde_json::json!(false), + "value2={value2} resp={response:?}" + ); + assert_eq!(queue_order(&test_state), vec![a, b]); + } + } + /// An empty or bare `SABnzbd_nzo_` id must not resolve to any job: /// `starts_with("")` is always true, which previously let a missing /// `value` pause or delete whichever job was first in the queue. @@ -3652,7 +4086,7 @@ mod tests { queue_job("default-job", "Default Job", "Default", JobStatus::Queued), queue_job("tv-job", "TV Job", "tv", JobStatus::Queued), ]; - let all = build_queue_response(&jobs, false, 0, 0, &SabApiRequest::default()); + let all = build_queue_response(&jobs, false, 0, 0, &SabApiRequest::default(), |_| 3); assert_eq!(all["queue"]["slots"][0]["cat"], "*"); assert_eq!(all["queue"]["slots"][1]["cat"], "tv"); @@ -3660,7 +4094,7 @@ mod tests { cat: Some("*".into()), ..SabApiRequest::default() }; - let filtered = build_queue_response(&jobs, false, 0, 0, &star); + let filtered = build_queue_response(&jobs, false, 0, 0, &star, |_| 3); assert_eq!(filtered["queue"]["noofslots"], 1); assert_eq!(filtered["queue"]["slots"][0]["filename"], "Default Job"); @@ -4348,9 +4782,18 @@ mod tests { // SABnzbd's cumulative 2 (+repair/unpack) is RustNZB's 3. let repair_unpack = addfile_multipart(SabApiRequest::default(), &[("pp", "2")]).await; assert_eq!(repair_unpack.pp_override, Some(3)); + // ... but without SABnzbd's "+Delete": the archives are kept. + assert_eq!(repair_unpack.delete_archives, Some(false)); + + let repair_unpack_delete = + addfile_multipart(SabApiRequest::default(), &[("pp", "3")]).await; + assert_eq!(repair_unpack_delete.pp_override, Some(3)); + assert_eq!(repair_unpack_delete.delete_archives, Some(true)); + assert_eq!(from_query.delete_archives, None); let invalid = addfile_multipart(SabApiRequest::default(), &[("pp", "7")]).await; assert_eq!(invalid.pp_override, None); + assert_eq!(invalid.delete_archives, None); let absent = addfile_multipart(SabApiRequest::default(), &[]).await; assert_eq!(absent.pp_override, None); @@ -4456,6 +4899,8 @@ mod tests { stages: Vec::new(), error_message: None, failure_code: None, + post_processing: None, + delete_archives: None, server_stats: Vec::new(), nzb_data: None, retry_data: None, @@ -4669,6 +5114,113 @@ mod tests { assert!(!history_ids(&test_state).contains(&"oldest-failed".to_string())); } + const RETRY_NZB: &[u8] = br#" + + + alt.binaries.test + retry-1@test + +"#; + + fn insert_retryable_history(test_state: &TestState, id: &str) { + let mut entry = history_entry(id, id, "tv", JobStatus::Failed, 1); + entry.output_dir = test_state.state.config().general.complete_dir.join(id); + entry.nzb_data = Some(RETRY_NZB.to_vec()); + test_state + .state + .queue_manager + .with_db(|database| database.history_insert(&entry).expect("insert history")); + } + + /// BUG-64: concurrent `mode=retry` calls for one history entry must + /// enqueue exactly one job; every caller gets that job's nzo_id. + #[tokio::test] + async fn concurrent_retries_of_one_history_entry_enqueue_one_job() { + let test_state = test_state(); + insert_retryable_history(&test_state, "failed-once"); + let barrier = std::sync::Barrier::new(5); + let runtime = tokio::runtime::Handle::current(); + let responses: Vec = std::thread::scope(|scope| { + let handles: Vec<_> = (0..5) + .map(|_| { + scope.spawn(|| { + let _runtime = runtime.enter(); + barrier.wait(); + handle_retry( + &test_state.state, + &SabApiRequest { + value: Some("failed-once".into()), + ..SabApiRequest::default() + }, + ) + .0 + }) + }) + .collect(); + handles.into_iter().map(|h| h.join().unwrap()).collect() + }); + + assert_eq!(test_state.state.queue_manager.get_jobs().len(), 1); + let first = &responses[0]["nzo_ids"][0]; + for response in &responses { + assert_eq!(response["status"], serde_json::json!(true), "{response}"); + assert_eq!(&response["nzo_ids"][0], first, "{response}"); + } + } + + /// Once the retried job has left the queue, the history entry can be + /// retried again. + #[tokio::test] + async fn retry_is_allowed_again_after_the_retried_job_leaves_the_queue() { + let test_state = test_state(); + insert_retryable_history(&test_state, "failed-twice"); + let req = SabApiRequest { + value: Some("failed-twice".into()), + ..SabApiRequest::default() + }; + let qm = &test_state.state.queue_manager; + let first = handle_retry(&test_state.state, &req).0; + assert_eq!(first["status"], serde_json::json!(true), "{first}"); + let job_id = qm.get_jobs()[0].id.clone(); + qm.remove_job(&job_id).expect("remove retried job"); + + let second = handle_retry(&test_state.state, &req).0; + assert_eq!(second["status"], serde_json::json!(true), "{second}"); + assert_ne!(second["nzo_ids"][0], first["nzo_ids"][0]); + assert_eq!(qm.get_jobs().len(), 1); + } + + /// `mode=retry_all` re-queues every retryable failed history entry and + /// answers like SABnzbd 5.0.4 `_api_retry_all`: `{"status": [nzo_ids]}`. + #[tokio::test] + async fn retry_all_requeues_every_failed_history_entry() { + let test_state = test_state(); + + let empty = dispatch_mode(&test_state.state, "retry_all", &SabApiRequest::default()).0; + assert_eq!(empty, serde_json::json!({ "status": [] })); + + for id in ["failed-one", "failed-two"] { + let mut entry = history_entry(id, id, "tv", JobStatus::Failed, 60); + entry.nzb_data = Some(SAMPLE_NZB.as_bytes().to_vec()); + test_state + .state + .queue_manager + .with_db(|database| database.history_insert(&entry).expect("insert history")); + } + insert_history_status(&test_state, "completed-one", JobStatus::Completed, 30); + + let response = dispatch_mode(&test_state.state, "retry_all", &SabApiRequest::default()).0; + let ids = response["status"].as_array().expect("status is a list"); + assert_eq!(ids.len(), 2, "response={response}"); + for id in ids { + assert!( + id.as_str().is_some_and(|id| id.starts_with("SABnzbd_nzo_")), + "response={response}" + ); + } + assert_eq!(test_state.state.queue_manager.get_jobs().len(), 2); + } + /// Without `del_files`, history delete only removes the DB record, as /// before. #[tokio::test] @@ -4975,21 +5527,33 @@ mod tests { #[test] fn addurl_job_name_sources_in_sabnzbd_order() { let url = reqwest::Url::parse("https://indexer.example/get/My%20File.nzb?id=1").unwrap(); - assert_eq!(addurl_job_name(None, None, &url), "My File"); + assert_eq!(addurl_job_name(None, None, &url).0, "My File"); assert_eq!( addurl_job_name( None, Some("attachment; filename*=UTF-8''Caf%C3%A9.nzb"), &url - ), + ) + .0, "Café" ); assert_eq!( - addurl_job_name(Some("Chosen.nzb"), Some("attachment; filename=x.nzb"), &url), + addurl_job_name(Some("Chosen.nzb"), Some("attachment; filename=x.nzb"), &url).0, "Chosen" ); + let braced = + reqwest::Url::parse("https://indexer.example/get/Path%7B%7Bpathpw%7D%7D.nzb").unwrap(); + assert_eq!( + addurl_job_name(Some("Chosen{{x}}"), None, &braced), + ("Chosen{{x}}".to_string(), Some("pathpw".to_string())), + "nzbname is split by the caller; the URL file name still supplies a password" + ); + assert_eq!( + addurl_job_name(None, None, &braced), + ("Path".to_string(), Some("pathpw".to_string())) + ); let bare = reqwest::Url::parse("https://indexer.example/").unwrap(); - assert!(!addurl_job_name(None, None, &bare).contains('/')); + assert!(!addurl_job_name(None, None, &bare).0.contains('/')); } /// `nzbname` also overrides the job name for uploads, from the query @@ -5140,4 +5704,187 @@ mod tests { }; assert_eq!(response.status(), StatusCode::BAD_REQUEST); } + + // -- SABnzbd inline job passwords (`name{{pw}}`, `name/pw`) ----------- + + const PASSWORD_META_NZB: &str = r#" + + meta-pw + + alt.binaries.test + + article1@example.com + + +"#; + + /// Multipart `mode=addfile` uploading `nzb` as `file_name`. + async fn addfile_named( + query: SabApiRequest, + fields: &[(&str, &str)], + file_name: &str, + nzb: &str, + ) -> NzbJob { + let TestState { state, _tempdir } = test_state(); + let state = Arc::new(state); + let boundary = "sabboundary"; + let mut body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"mode\"\r\n\r\naddfile\r\n" + ); + for (name, value) in fields { + body.push_str(&format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n" + )); + } + body.push_str(&format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"name\"; filename=\"{file_name}\"\r\nContent-Type: application/x-nzb\r\n\r\n{nzb}\r\n--{boundary}--\r\n" + )); + let query = SabApiRequest { + apikey: Some("contract-api-key".into()), + ..query + }; + let request = Request::builder() + .method("POST") + .uri("/sabnzbd/api") + .header( + CONTENT_TYPE, + format!("multipart/form-data; boundary={boundary}"), + ) + .body(axum::body::Body::from(body)) + .expect("build request"); + let response = h_sabnzbd_api_post(State(state.clone()), Query(query), request) + .await + .expect("addfile over multipart") + .into_response(); + let value = json_body(response).await; + assert_eq!(value["status"], serde_json::json!(true), "resp={value}"); + let mut jobs = state.queue_manager.get_jobs(); + assert_eq!(jobs.len(), 1); + jobs.remove(0) + } + + /// SABnzbd moves `{{pw}}` and a trailing `/pw` out of `nzbname` into the + /// job password; the braces never reach the job or folder name. + #[tokio::test] + async fn addfile_takes_inline_password_from_nzbname() { + for nzbname in ["MyShow{{INLINEPW}}", "MyShow/INLINEPW", "MyShow / INLINEPW"] { + let job = addfile_named( + SabApiRequest { + nzbname: Some(nzbname.into()), + ..SabApiRequest::default() + }, + &[], + "upload.nzb", + SAMPLE_NZB, + ) + .await; + assert_eq!(job.name, "MyShow", "{nzbname}"); + assert_eq!(job.password.as_deref(), Some("INLINEPW"), "{nzbname}"); + let folder = job.output_dir.file_name().unwrap().to_string_lossy(); + assert!(!folder.contains('{'), "{nzbname}: {folder}"); + } + } + + /// The uploaded file name carries the password the same way. + #[tokio::test] + async fn addfile_takes_inline_password_from_file_name() { + let job = addfile_named( + SabApiRequest::default(), + &[], + "My.Release{{filepw}}.nzb", + SAMPLE_NZB, + ) + .await; + assert_eq!(job.name, "My.Release"); + assert_eq!(job.password.as_deref(), Some("filepw")); + + // nzbname names the job; the file name still supplies the password. + let job = addfile_named( + SabApiRequest { + nzbname: Some("Chosen".into()), + ..SabApiRequest::default() + }, + &[], + "My.Release{{filepw}}.nzb", + SAMPLE_NZB, + ) + .await; + assert_eq!(job.name, "Chosen"); + assert_eq!(job.password.as_deref(), Some("filepw")); + } + + /// Explicit `password` > inline password > NZB ``. + #[tokio::test] + async fn addfile_password_precedence() { + let inline = addfile_named( + SabApiRequest { + nzbname: Some("Show{{inline-pw}}".into()), + ..SabApiRequest::default() + }, + &[], + "upload.nzb", + PASSWORD_META_NZB, + ) + .await; + assert_eq!(inline.password.as_deref(), Some("inline-pw")); + + let explicit = addfile_named( + SabApiRequest { + nzbname: Some("Show{{inline-pw}}".into()), + ..SabApiRequest::default() + }, + &[("password", "explicit-pw")], + "upload.nzb", + PASSWORD_META_NZB, + ) + .await; + assert_eq!(explicit.name, "Show"); + assert_eq!(explicit.password.as_deref(), Some("explicit-pw")); + + let meta = + addfile_named(SabApiRequest::default(), &[], "Show.nzb", PASSWORD_META_NZB).await; + assert_eq!(meta.password.as_deref(), Some("meta-pw")); + } + + /// `addurl` applies the convention to `nzbname` and to the name taken + /// from `Content-Disposition`. + #[tokio::test] + async fn addurl_takes_inline_password() { + async fn add(url: String, nzbname: Option<&str>) -> NzbJob { + let TestState { state, _tempdir } = test_state(); + { + let mut config = (*state.config()).clone(); + config.general.fetch_allowed_hosts = vec!["127.0.0.1".into()]; + state.config.store(std::sync::Arc::new(config)); + } + let state = Arc::new(state); + let req = SabApiRequest { + mode: Some("addurl".into()), + name: Some(url), + nzbname: nzbname.map(str::to_string), + apikey: Some("contract-api-key".into()), + ..SabApiRequest::default() + }; + let response = h_sabnzbd_api_get(State(state.clone()), Query(req)) + .await + .expect("addurl over GET") + .into_response(); + let value = json_body(response).await; + assert_eq!(value["status"], serde_json::json!(true), "resp={value}"); + state.queue_manager.get_jobs().remove(0) + } + + let job = add(spawn_nzb_server(SAMPLE_NZB).await, Some("Url.Show/urlpw")).await; + assert_eq!(job.name, "Url.Show"); + assert_eq!(job.password.as_deref(), Some("urlpw")); + + let url = spawn_nzb_server_with_headers( + SAMPLE_NZB, + "Content-Disposition: attachment; filename=\"Disp.Show{{disppw}}.nzb\"\r\n", + ) + .await; + let job = add(url, None).await; + assert_eq!(job.name, "Disp.Show"); + assert_eq!(job.password.as_deref(), Some("disppw")); + } } diff --git a/crates/nzb-web/src/startup.rs b/crates/nzb-web/src/startup.rs index 08ae0abf..b47fc7c1 100644 --- a/crates/nzb-web/src/startup.rs +++ b/crates/nzb-web/src/startup.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::Context; use arc_swap::ArcSwap; -use tracing::info; +use tracing::{info, warn}; use crate::nzb_core::config::AppConfig; use crate::nzb_core::db::Database; @@ -49,6 +49,22 @@ fn env_flag_enabled(name: &str) -> Option { /// Create a data directory (e.g. `data_dir`/`incomplete_dir`/`complete_dir`), attaching /// the failing path and a permission hint to any error so failures are actionable /// instead of a bare `Permission denied (os error 13)`. +/// A hand-edited config.toml can carry a category the API would refuse. +/// Starting anyway keeps the rest of the service usable; the warning +/// explains why jobs in that category fail to enqueue. +fn warn_unsafe_categories(config: &AppConfig) { + for category in &config.categories { + if let Err(reason) = category.validate() { + warn!( + category = %category.name, + output_dir = ?category.output_dir, + reason, + "Unsafe category in config; jobs in it will fail to enqueue until it is fixed" + ); + } + } +} + fn create_data_dir(path: &Path) -> anyhow::Result<()> { std::fs::create_dir_all(path).with_context(|| { format!( @@ -135,6 +151,8 @@ pub async fn initialize( config.otel.service_name = val; } + warn_unsafe_categories(&config); + // Ensure directories exist create_data_dir(&config.general.data_dir)?; create_data_dir(&config.general.incomplete_dir)?; @@ -216,18 +234,9 @@ pub async fn initialize( // without a restart. let shared_config = Arc::new(ArcSwap::new(Arc::new(config))); - // Always start the RSS monitor so feeds added later via the API are polled. - let data_dir_for_rss = shared_config.load().general.data_dir.clone(); - let monitor = crate::rss_monitor::RssMonitor::new( - Arc::clone(&shared_config), - Arc::clone(&queue_manager), - data_dir_for_rss, - ); - tokio::spawn(async move { monitor.run().await }); - // Build shared application state let state = Arc::new(AppState::new( - shared_config, + Arc::clone(&shared_config), config_path, Arc::clone(&queue_manager), log_buffer.clone(), @@ -235,6 +244,18 @@ pub async fn initialize( credential_store, )); + // Always start the RSS monitor so feeds added later via the API are polled. + // Every committed config change wakes it, so a new or re-enabled feed is + // checked promptly instead of after the monitor's current sleep. + let data_dir_for_rss = shared_config.load().general.data_dir.clone(); + let monitor = crate::rss_monitor::RssMonitor::new( + shared_config, + Arc::clone(&queue_manager), + data_dir_for_rss, + ) + .with_wake(Arc::clone(&state.rss_monitor_wake)); + tokio::spawn(async move { monitor.run().await }); + Ok(StartupResult { state, queue_manager, diff --git a/crates/nzb-web/src/state.rs b/crates/nzb-web/src/state.rs index 02a5dee2..f3543035 100644 --- a/crates/nzb-web/src/state.rs +++ b/crates/nzb-web/src/state.rs @@ -23,6 +23,10 @@ pub struct AppState { /// Serialises config writers so a read-modify-write cycle (including /// persisting the TOML) cannot interleave with another one. config_write: Mutex<()>, + /// Signalled after every committed config change so the RSS monitor + /// re-reads its feeds at once instead of finishing its current sleep. + /// Single consumer: `notify_one` stores a permit if the monitor is busy. + pub rss_monitor_wake: Arc, } impl AppState { @@ -43,6 +47,7 @@ impl AppState { credential_store, started_at: std::time::Instant::now(), config_write: Mutex::new(()), + rss_monitor_wake: Arc::new(tokio::sync::Notify::new()), } } @@ -84,6 +89,7 @@ impl AppState { fn commit_config(&self, config: AppConfig) -> anyhow::Result<()> { config.save(&self.config_path)?; self.config.store(Arc::new(config)); + self.rss_monitor_wake.notify_one(); Ok(()) } } diff --git a/crates/nzb-web/tests/harness_incomplete_cleanup.rs b/crates/nzb-web/tests/harness_incomplete_cleanup.rs index f5b92f61..69cf5a46 100644 --- a/crates/nzb-web/tests/harness_incomplete_cleanup.rs +++ b/crates/nzb-web/tests/harness_incomplete_cleanup.rs @@ -144,6 +144,8 @@ fn history_entry(id: &str, complete: &Path) -> HistoryEntry { stages: Vec::new(), error_message: Some("failed".to_string()), failure_code: Some(JobFailureCode::DownloadFailed), + post_processing: None, + delete_archives: None, server_stats: Vec::new(), nzb_data: None, retry_data: None, diff --git a/crates/nzb-web/tests/workflow_fixtures.rs b/crates/nzb-web/tests/workflow_fixtures.rs index d57ca104..d93dd5ef 100644 --- a/crates/nzb-web/tests/workflow_fixtures.rs +++ b/crates/nzb-web/tests/workflow_fixtures.rs @@ -255,3 +255,27 @@ async fn slowly_written_nzb_is_parsed_only_once_complete() { assert!(!watch_dir.join("failed/slow.nzb").exists()); assert!(watch_dir.join("processed/slow.nzb").exists()); } + +/// SABnzbd's `name{{password}}` file-name convention applies to the watch +/// folder: the password is split off before the job is named. +#[tokio::test] +async fn watched_nzb_file_name_carries_inline_password() { + let temp = tempfile::tempdir().unwrap(); + let watch_dir = temp.path().join("watch"); + std::fs::create_dir_all(&watch_dir).unwrap(); + std::fs::write( + watch_dir.join("Watched.Show{{watchpw}}.nzb"), + fixture_nzb("pw.bin"), + ) + .unwrap(); + + let queue = watch_queue(temp.path()); + let watcher_task = tokio::spawn(DirWatcher::new(watch_dir.clone(), queue.clone()).run()); + let imported = wait_until(Duration::from_secs(2), || queue.queue_size() == 1).await; + watcher_task.abort(); + + assert!(imported, "watch folder did not enqueue the NZB"); + let job = queue.get_jobs().remove(0); + assert_eq!(job.name, "Watched.Show"); + assert_eq!(job.password.as_deref(), Some("watchpw")); +} diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 31063d98..eb5ea881 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -3015,7 +3015,7 @@ dependencies = [ [[package]] name = "nzb-core" -version = "0.2.18" +version = "0.2.19" dependencies = [ "anyhow", "chrono", @@ -3109,7 +3109,7 @@ dependencies = [ [[package]] name = "nzb-web" -version = "0.4.23" +version = "0.4.24" dependencies = [ "anyhow", "arc-swap", diff --git a/docs/KNOWN_ISSUES.md b/docs/KNOWN_ISSUES.md index ebeb77f3..020d1756 100644 --- a/docs/KNOWN_ISSUES.md +++ b/docs/KNOWN_ISSUES.md @@ -11,3 +11,22 @@ methodology limits that must be addressed or disclosed before publishing any comparison. The current baseline, validation gate, and accepted/rejected optimization experiments are recorded in the [performance status](PERFORMANCE_STATUS.md). + +## Archive passwords + +- **Command-line exposure with older extractors.** rustnzb writes a job's + archive password to the extractor's standard input, so it does not appear + in `ps` or `/proc//cmdline`, when the extractor is rarlab UNRAR/RAR 6 + or later or 7-Zip 21 or later (the container image ships both). Other + extractors — older unrar releases, `unrar-free`, and p7zip 16.02 (the `7z` + from Debian's `p7zip-full`, which reads passwords from the terminal) — still + receive it as a `-p` argument, where any local user can read it + while the extraction runs. So does a password that contains a line break. + Install a current unrar or 7-Zip, or mount `/proc` with `hidepid=2`, on + multi-user hosts. When both are installed, rustnzb runs 7-Zip's `7zz` in + preference to `7z`. +- **Stored in plaintext.** Extraction needs the original password, so it is + kept unencrypted in the queue database (`queue.password` in the SQLite + database under the data directory) and in the stored NZB. Keep the data + directory readable only by the rustnzb user. The SABnzbd-compatible queue + returns it in each slot's `password`, as SABnzbd does. diff --git a/e2e/tests/first-boot.spec.ts b/e2e/tests/first-boot.spec.ts index 71017846..2546fb54 100644 --- a/e2e/tests/first-boot.spec.ts +++ b/e2e/tests/first-boot.spec.ts @@ -156,7 +156,9 @@ test.describe.serial('First boot', () => { await page.getByRole('button', { name: 'Import from SABnzbd' }).click(); await expect(page.getByRole('button', { name: /live instance/i })).toBeVisible(); - // Fill in an unreachable URL and a dummy API key + // A loopback URL. The fetch guard refuses private addresses before any + // connection attempt, and the page shows that reason (human_readable) + // rather than the generic fallback. const urlField = page.getByPlaceholder(/http.*localhost.*8080/i); await urlField.fill('http://localhost:1'); @@ -166,7 +168,7 @@ test.describe.serial('First boot', () => { await page.getByRole('button', { name: /fetch config/i }).click(); // Wait for the error — connecting to :1 will fail quickly - await expect(page.getByText(/failed to connect/i)).toBeVisible({ timeout: 15000 }); + await expect(page.getByText(/private\/reserved address|failed to connect/i)).toBeVisible({ timeout: 15000 }); // Still on connect step await expect(page.getByRole('button', { name: /fetch config/i })).toBeVisible(); });