From cf882bb41eafa92f66e570facad1fb0a008a78f5 Mon Sep 17 00:00:00 2001 From: "Sangamesh V." <60685551+SangameshV@users.noreply.github.com> Date: Thu, 20 Aug 2026 21:54:06 +0530 Subject: [PATCH 01/37] chore: Add FoD MFA support (#1073) feat: `fcli fod session login`: Add `--code` and `--totp` options to support MFA-based logins (resolves #1059) --- .github/instructions/java.instructions.md | 7 +- ...CPServerHttpSessionDescriptorResolver.java | 39 ++------- .../cli/cmd/FoDSessionLoginCommand.java | 57 ++++++++++-- .../cli/mixin/FoDSessionLoginOptions.java | 87 ++++++++++--------- .../session/helper/oauth/FoDOAuthHelper.java | 22 +++-- .../helper/oauth/IFoDUserAuthCode.java | 18 ++++ .../oauth/impl/BasicFoDUserAuthCode.java | 30 +++++++ .../oauth/impl/BasicFoDUserCredentials.java | 31 +++++++ .../cli/fod/i18n/FoDMessages.properties | 2 + 9 files changed, 205 insertions(+), 88 deletions(-) create mode 100644 fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/IFoDUserAuthCode.java create mode 100644 fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/impl/BasicFoDUserAuthCode.java create mode 100644 fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/impl/BasicFoDUserCredentials.java diff --git a/.github/instructions/java.instructions.md b/.github/instructions/java.instructions.md index 0ddc7742e3c..6b0b54a46d2 100644 --- a/.github/instructions/java.instructions.md +++ b/.github/instructions/java.instructions.md @@ -30,16 +30,19 @@ Always use `headerReplace(name, value)` — never `accept()`, `contentType()`, o - New/updated code should throw only fcli-domain exceptions (`Fcli*Exception`), module-domain exceptions (for example `Aviator*Exception` in Aviator modules), or picocli exceptions (`ParameterException` and related) when integrating with command parsing. - Avoid throwing standard Java runtime exceptions (`IllegalArgumentException`, `IllegalStateException`, `RuntimeException`, and similar) for user-facing or command-flow errors. +- **Checked exceptions** (e.g., `IOException`, `JsonProcessingException`): Wrap in `FcliTechnicalException` to preserve the cause chain. +- **Runtime exceptions** (e.g., `UnexpectedHttpResponseException`): Re-throw as-is unless special handling is needed (e.g., a specific error code requires a user-friendly message). Avoid unnecessary wrapping to keep stack traces short and relevant. | Scenario | Exception | |----------|-----------| | Invalid/missing user input | `FcliSimpleException` | | External resource not found | `FcliSimpleException` with remediation | | User abort | `FcliAbortedByUserException` | -| I/O, network, JSON parse | `FcliTechnicalException` (wrap cause) | +| Checked exception (I/O, JSON parse) | `FcliTechnicalException` (wrap cause) | +| Runtime exception (no special handling needed) | Re-throw as-is | | Invariant violation, unreachable | `FcliBugException` | -Messages: actionable, sentence case, no trailing periods. Preserve root cause in wrapping. +Messages: actionable, sentence case, no trailing periods. Wrap root cause only for checked exceptions. ## Design Patterns diff --git a/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/helper/http/MCPServerHttpSessionDescriptorResolver.java b/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/helper/http/MCPServerHttpSessionDescriptorResolver.java index 8a37a37baa0..c7bd5d5059f 100644 --- a/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/helper/http/MCPServerHttpSessionDescriptorResolver.java +++ b/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/helper/http/MCPServerHttpSessionDescriptorResolver.java @@ -41,7 +41,7 @@ import com.fortify.cli.fod._common.session.helper.oauth.FoDOAuthHelper; import com.fortify.cli.fod._common.session.helper.oauth.FoDTokenCreateResponse; import com.fortify.cli.fod._common.session.helper.oauth.IFoDClientCredentials; -import com.fortify.cli.fod._common.session.helper.oauth.IFoDUserCredentials; +import com.fortify.cli.fod._common.session.helper.oauth.impl.BasicFoDUserCredentials; import com.fortify.cli.ssc._common.session.cli.mixin.SSCAndScanCentralSessionLoginOptions.SSCAndScanCentralUrlConfigOptions.SSCComponentDisable; import com.fortify.cli.ssc._common.session.helper.ISSCAndScanCentralCredentialsConfig; import com.fortify.cli.ssc._common.session.helper.ISSCAndScanCentralUrlConfig; @@ -348,11 +348,11 @@ private FoDTokenCreateResponse createFoDTokenResponse(ParsedAuthorization auth, try { return FoDOAuthHelper.createToken( urlConfig, - new HttpMcpFoDUserCredentials( - auth.fodTenant(), - auth.fodUser(), - pwd - ), + BasicFoDUserCredentials.builder() + .tenant(auth.fodTenant()) + .user(auth.fodUser()) + .password(pwd) + .build(), DEFAULT_FOD_SCOPES ); } finally { @@ -380,33 +380,6 @@ public String getClientSecret() { } } - private static final class HttpMcpFoDUserCredentials implements IFoDUserCredentials { - private final String tenant; - private final String user; - private final char[] password; - - private HttpMcpFoDUserCredentials(String tenant, String user, char[] password) { - this.tenant = tenant; - this.user = user; - this.password = password; - } - - @Override - public String getUser() { - return user; - } - - @Override - public char[] getPassword() { - return password; - } - - @Override - public String getTenant() { - return tenant; - } - } - private static final class HttpMcpSscUrlConfig implements ISSCAndScanCentralUrlConfig { private final MCPServerHttpConfig.SscConfig config; diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/cli/cmd/FoDSessionLoginCommand.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/cli/cmd/FoDSessionLoginCommand.java index b6f9bbcd016..2daed82ee59 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/cli/cmd/FoDSessionLoginCommand.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/cli/cmd/FoDSessionLoginCommand.java @@ -14,6 +14,7 @@ import com.fortify.cli.common.exception.FcliSimpleException; import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; +import com.fortify.cli.common.rest.unirest.UnexpectedHttpResponseException; import com.fortify.cli.common.rest.unirest.config.IUrlConfig; import com.fortify.cli.common.session.cli.cmd.AbstractSessionLoginCommand; import com.fortify.cli.common.session.cli.mixin.ISessionNameSupplier; @@ -34,12 +35,31 @@ public class FoDSessionLoginCommand extends AbstractSessionLoginCommand (or -c ) to provide the security code\n" + + " --totp to indicate the code is from a TOTP authenticator app"; + + private static final String ERROR_WITH_CODE = "Authentication failed. Possible causes:\n" + + " - Incorrect username or password\n" + + " - MFA/TOTP code incorrect, expired, or wrong type (TOTP vs MFA)\n" + + "Please verify your credentials and MFA/TOTP code if applicable:\n" + + MFA_GUIDANCE; + + private static final String ERROR_WITHOUT_CODE = "Authentication failed. Possible causes:\n" + + " - Incorrect username or password\n" + + " - FoD tenant requires MFA/TOTP authentication\n\n" + + MFA_GUIDANCE; + + private static final String ERROR_CLIENT_CREDENTIALS = "Authentication failed. Possible causes:\n" + + " - Incorrect client ID or client secret\n" + + " - Client credentials have expired"; + @Override public ISessionNameSupplier getSessionNameSupplier() { return unirestInstanceSupplierMixin; } - + @Override protected void logoutBeforeNewLogin(String sessionName, FoDSessionDescriptor sessionDescriptor) { unirestInstanceSupplierMixin.close(sessionName); @@ -48,17 +68,36 @@ protected void logoutBeforeNewLogin(String sessionName, FoDSessionDescriptor ses @Override protected FoDSessionDescriptor login(String sessionName) { - FoDSessionDescriptor sessionDescriptor; + FoDSessionDescriptor sessionDescriptor = null; IUrlConfig urlConfig = loginOptions.getUrlConfigOptions(); - if ( loginOptions.hasClientCredentials() ) { - FoDTokenCreateResponse createTokenResponse = FoDOAuthHelper.createToken(urlConfig, loginOptions.getClientCredentialOptions(), loginOptions.getAuthOptions().getScopes()); - sessionDescriptor = new FoDSessionDescriptor(urlConfig, createTokenResponse); - } else if ( loginOptions.hasUserCredentials() ) { - FoDTokenCreateResponse createTokenResponse = FoDOAuthHelper.createToken(urlConfig, loginOptions.getUserCredentials(), loginOptions.getAuthOptions().getScopes()); - sessionDescriptor = new FoDSessionDescriptor(urlConfig, createTokenResponse); + if (loginOptions.hasClientCredentials()) { + try { + FoDTokenCreateResponse createTokenResponse = FoDOAuthHelper.createToken(urlConfig, + loginOptions.getClientCredentialOptions(), loginOptions.getAuthOptions().getScopes()); + sessionDescriptor = new FoDSessionDescriptor(urlConfig, createTokenResponse); + } catch (UnexpectedHttpResponseException e) { + handleUnexpectedHttpResponseException(e, ERROR_CLIENT_CREDENTIALS); + } + } else if (loginOptions.hasUserCredentials()) { + try { + FoDTokenCreateResponse createTokenResponse = FoDOAuthHelper.createToken(urlConfig, + loginOptions.getUserCredentials(), loginOptions.getAuthCode(), + loginOptions.getAuthOptions().getScopes()); + sessionDescriptor = new FoDSessionDescriptor(urlConfig, createTokenResponse); + } catch (UnexpectedHttpResponseException e) { + String errorMessage = loginOptions.hasSecurityCode() ? ERROR_WITH_CODE : ERROR_WITHOUT_CODE; + handleUnexpectedHttpResponseException(e, errorMessage); + } } else { throw new FcliSimpleException("Either FoD client or user credentials must be provided"); } return sessionDescriptor; } + + private void handleUnexpectedHttpResponseException(UnexpectedHttpResponseException e, String msg) { + if (e.getStatus() == 400) { + throw new FcliSimpleException(msg); + } + throw e; + } } diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/cli/mixin/FoDSessionLoginOptions.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/cli/mixin/FoDSessionLoginOptions.java index 3250f0e6596..484ef85b831 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/cli/mixin/FoDSessionLoginOptions.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/cli/mixin/FoDSessionLoginOptions.java @@ -22,7 +22,10 @@ import com.fortify.cli.common.session.cli.mixin.UserCredentialOptions; import com.fortify.cli.fod._common.rest.helper.FoDProductHelper; import com.fortify.cli.fod._common.session.helper.oauth.IFoDClientCredentials; +import com.fortify.cli.fod._common.session.helper.oauth.IFoDUserAuthCode; import com.fortify.cli.fod._common.session.helper.oauth.IFoDUserCredentials; +import com.fortify.cli.fod._common.session.helper.oauth.impl.BasicFoDUserAuthCode; +import com.fortify.cli.fod._common.session.helper.oauth.impl.BasicFoDUserCredentials; import lombok.Getter; import lombok.SneakyThrows; @@ -33,28 +36,33 @@ public class FoDSessionLoginOptions { @Mixin @Getter private FoDUrlConfigOptions urlConfigOptions = new FoDUrlConfigOptions(); - + @ArgGroup(exclusive = false, multiplicity = "1", order = 2) @Getter private FoDAuthOptions authOptions = new FoDAuthOptions(); - + public static class FoDAuthOptions { @ArgGroup(exclusive = true, multiplicity = "1", order = 3) @Getter private FoDCredentialOptions credentialOptions = new FoDCredentialOptions(); @Option(names="--scopes", defaultValue="api-tenant", split=",") @Getter private String[] scopes; } - + public static class FoDCredentialOptions { - @ArgGroup(exclusive = false, multiplicity = "1", order = 1) + @ArgGroup(exclusive = false, multiplicity = "1", order = 1) @Getter private FoDUserCredentialOptions userCredentialOptions = new FoDUserCredentialOptions(); - @ArgGroup(exclusive = false, multiplicity = "1", order = 2) + @ArgGroup(exclusive = false, multiplicity = "1", order = 2) @Getter private FoDClientCredentialOptions clientCredentialOptions = new FoDClientCredentialOptions(); } - + public static class FoDUserCredentialOptions extends UserCredentialOptions { @Option(names = {"-t", "--tenant"}, required = true) @MaskValue(sensitivity = LogSensitivityLevel.low, description = "FOD TENANT") @Getter private String tenant; + @Option(names = {"--code", "-c" }, paramLabel = "", arity = "0..1", interactive = true, echo = false) + @MaskValue(sensitivity = LogSensitivityLevel.low, description = "FOD TOTP/MFA CODE") + @Getter private String securityCode; + @Option(names = {"--totp" }) + @Getter private boolean isTotp; } public static class FoDClientCredentialOptions implements IFoDClientCredentials { @@ -72,7 +80,7 @@ public FoDUserCredentialOptions getUserCredentialOptions() { .map(FoDCredentialOptions::getUserCredentialOptions) .orElse(null); } - + public FoDClientCredentialOptions getClientCredentialOptions() { return Optional.ofNullable(authOptions) .map(FoDAuthOptions::getCredentialOptions) @@ -89,55 +97,56 @@ public final boolean hasUserCredentials() { && userCredentialOptions.getPassword().length > 0; } - public final BasicFoDUserCredentials getUserCredentials() { + public final IFoDUserCredentials getUserCredentials() { var u = getUserCredentialOptions(); - return BasicFoDUserCredentials.builder().tenant(u.getTenant()).user(u.getUser()).password(u.getPassword()).build(); + return BasicFoDUserCredentials.builder() + .tenant(u.getTenant()) + .user(u.getUser()) + .password(u.getPassword()) + .build(); } - + public final boolean hasClientCredentials() { FoDClientCredentialOptions clientCredentialOptions = getClientCredentialOptions(); return clientCredentialOptions!=null && StringUtils.isNotBlank(clientCredentialOptions.getClientId()) && StringUtils.isNotBlank(clientCredentialOptions.getClientSecret()); } - + + public boolean hasSecurityCode() { + var userCred = getUserCredentialOptions(); + return userCred != null && StringUtils.isNotBlank(userCred.getSecurityCode()); + } + + public String getSecurityCode() { + var userCred = getUserCredentialOptions(); + return userCred != null ? userCred.getSecurityCode() : null; + } + + public boolean isTotp() { + var userCred = getUserCredentialOptions(); + return userCred != null && userCred.isTotp(); + } + + public IFoDUserAuthCode getAuthCode() { + var u = getUserCredentialOptions(); + if (u == null || StringUtils.isBlank(u.getSecurityCode())) { return null; } + return BasicFoDUserAuthCode.builder() + .securityCode(u.getSecurityCode()) + .isTotp(u.isTotp()) + .build(); + } + @Command public static final class FoDUrlConfigOptions extends UrlConfigOptions { @Override @SneakyThrows public String getUrl() { return FoDProductHelper.INSTANCE.getApiUrl(super.getUrl()); } - + @Override protected int getDefaultSocketTimeoutInMillis() { return 600000; } } - - /** - * Basic immutable FoD user credentials with builder pattern. - */ - public static final class BasicFoDUserCredentials implements IFoDUserCredentials { - private final String tenant; - private final String user; - private final char[] password; - private BasicFoDUserCredentials(Builder b) { - this.tenant = b.tenant; - this.user = b.user; - this.password = b.password; - } - public static Builder builder() { return new Builder(); } - @Override public String getTenant() { return tenant; } - @Override public String getUser() { return user; } - @Override public char[] getPassword() { return password; } - public static final class Builder { - private String tenant; private String user; private char[] password; - public Builder tenant(String tenant){ this.tenant=tenant; return this; } - public Builder user(String user){ this.user=user; return this; } - public Builder password(char[] password){ this.password=password; return this; } - public BasicFoDUserCredentials build(){ - return new BasicFoDUserCredentials(this); - } - } - } } diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/FoDOAuthHelper.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/FoDOAuthHelper.java index 7dd2e21a28e..90ed7e0f0e4 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/FoDOAuthHelper.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/FoDOAuthHelper.java @@ -28,20 +28,28 @@ // TODO Consider moving all classes in this package to a more appropriate package, // for example as a sub-package of the 'rest' package. public class FoDOAuthHelper { + public static final FoDTokenCreateResponse createToken(IUrlConfig urlConfig, IFoDUserCredentials uc, String... scopes) { - Map formData = generateTokenRequest(uc, scopes); - try ( var unirest = UnirestHelper.createUnirestInstance() ) { + Map formData = generateTokenRequest(uc, null, scopes); + try ( var unirest = UnirestHelper.createUnirestInstance() ) { + return createToken(unirest, urlConfig, formData); + } + } + + public static final FoDTokenCreateResponse createToken(IUrlConfig urlConfig, IFoDUserCredentials uc, IFoDUserAuthCode authCode, String... scopes) { + Map formData = generateTokenRequest(uc, authCode, scopes); + try ( var unirest = UnirestHelper.createUnirestInstance() ) { return createToken(unirest, urlConfig, formData); } } public static final FoDTokenCreateResponse createToken(IUrlConfig urlConfig, IFoDClientCredentials cc, String... scopes) { Map formData = generateTokenRequest(cc, scopes); - try ( var unirest = UnirestHelper.createUnirestInstance() ) { + try ( var unirest = UnirestHelper.createUnirestInstance() ) { return createToken(unirest, urlConfig, formData); } } - + private static final FoDTokenCreateResponse createToken(UnirestInstance unirest, IUrlConfig urlConfig, Map formData) { configureUnirest(unirest, urlConfig); return unirest.post("/oauth/token") @@ -60,12 +68,16 @@ private static final void configureUnirest(UnirestInstance unirest, IUrlConfig u UnirestJsonHeaderConfigurer.configure(unirest); } - private static final Map generateTokenRequest(IFoDUserCredentials uc, String... scopes) { + private static final Map generateTokenRequest(IFoDUserCredentials uc, IFoDUserAuthCode authCode, String... scopes) { Map result = new LinkedHashMap<>(); result.put("scope", String.join(",", scopes)); result.put("grant_type", "password"); result.put("username", String.format("%s\\%s", uc.getTenant(), uc.getUser())); result.put("password", String.valueOf(uc.getPassword())); + if (null != authCode && null != authCode.getSecurityCode()) { + result.put("security_code", authCode.getSecurityCode()); + result.put("do_totp", authCode.isTotp()); + } return result; } diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/IFoDUserAuthCode.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/IFoDUserAuthCode.java new file mode 100644 index 00000000000..2a2daa088f0 --- /dev/null +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/IFoDUserAuthCode.java @@ -0,0 +1,18 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.fod._common.session.helper.oauth; + +public interface IFoDUserAuthCode { + default String getSecurityCode() { return null; } + default boolean isTotp() { return false; } +} diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/impl/BasicFoDUserAuthCode.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/impl/BasicFoDUserAuthCode.java new file mode 100644 index 00000000000..14396eb81fa --- /dev/null +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/impl/BasicFoDUserAuthCode.java @@ -0,0 +1,30 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.fod._common.session.helper.oauth.impl; + +import com.fortify.cli.fod._common.session.helper.oauth.IFoDUserAuthCode; + +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Basic immutable FoD user auth code with builder pattern. + */ +@Builder +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public final class BasicFoDUserAuthCode implements IFoDUserAuthCode { + @Getter private final String securityCode; + @Getter private final boolean isTotp; +} diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/impl/BasicFoDUserCredentials.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/impl/BasicFoDUserCredentials.java new file mode 100644 index 00000000000..f7a0834327a --- /dev/null +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/session/helper/oauth/impl/BasicFoDUserCredentials.java @@ -0,0 +1,31 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.fod._common.session.helper.oauth.impl; + +import com.fortify.cli.fod._common.session.helper.oauth.IFoDUserCredentials; + +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Basic immutable FoD user credentials with builder pattern. + */ +@Builder +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public final class BasicFoDUserCredentials implements IFoDUserCredentials { + @Getter private final String tenant; + @Getter private final String user; + @Getter private final char[] password; +} diff --git a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties index 62dcf5b6c6d..15976845f03 100644 --- a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties +++ b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties @@ -123,6 +123,8 @@ fcli.fod.session.login.client-secret = FoD client secret. fcli.fod.session.login.scopes = FoD scopes to request. Default value: ${DEFAULT-VALUE} fcli.fod.session.login.fod-session = Name for this FoD session. Default value: ${DEFAULT-VALUE}. fcli.fod.session.login.header = Repeatable option to add custom HTTP headers in requests to FoD for this session, in format `NAME: VALUE`. +fcli.fod.session.login.code = Security code (TOTP from authenticator or MFA code from email/SMS). +fcli.fod.session.login.totp = Indicates the provided code is TOTP from authenticator app (sets do_totp=true). fcli.fod.session.logout.usage.header = Terminate FoD session. fcli.fod.session.logout.usage.description = This command terminates an FoD session previously created \ From e63c814e531c5aab8748a145dff4000c0c21913e Mon Sep 17 00:00:00 2001 From: Kireeti Varma <81861747+kireetivar@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:15:32 +0530 Subject: [PATCH 02/37] chore: Add Aviator source encoding handling (#1066) feat: `fcli aviator ssc audit`: Add `--source-encodings` option for source decoding and audit skip reporting feat: `fcli aviator ssc apply-remediations`: Add `--source-encodings` option for source file decoding and encoding feat: `fcli fod aviator apply-remediations`: Add `--source-encodings` option for source file decoding and encoding Co-authored-by: Ankit Rathod --- fcli-core/fcli-app/build.gradle.kts | 2 +- .../cli/converter/SourceDecoderConverter.java | 36 +++ .../cli/mixin/SourceEncodingsMixin.java | 44 +++ .../ApplyAutoRemediationOnSource.java | 14 +- .../fortify/cli/aviator/audit/AuditFPR.java | 106 +++++-- .../cli/aviator/audit/IssueAuditor.java | 20 +- .../aviator/audit/model/AuditFprOptions.java | 3 + .../aviator/audit/model/AuditResponse.java | 31 +- .../aviator/audit/model/FPRAuditResult.java | 27 ++ .../aviator/fpr/processor/AuditProcessor.java | 68 ++++- .../fpr/processor/RemediationProcessor.java | 143 ++++------ .../fpr/processor/StreamingFVDLProcessor.java | 19 +- .../fpr/utils/CharsetSourceDecoder.java | 45 +++ .../fpr/utils/CompositeSourceDecoder.java | 53 ++++ .../cli/aviator/fpr/utils/FileUtils.java | 54 +++- .../aviator/fpr/utils/FprSourceDecoder.java | 57 ++++ .../cli/aviator/fpr/utils/ISourceDecoder.java | 55 ++++ .../aviator/fpr/utils/SourceCodeEnricher.java | 119 +++++--- .../cli/aviator/fpr/utils/SourceDecoders.java | 82 ++++++ .../cli/aviator/fpr/utils/SourceEncoder.java | 64 +++++ .../cli/aviator/grpc/AviatorGrpcClient.java | 9 +- .../aviator/grpc/AviatorStreamProcessor.java | 152 ++++++++-- .../fortify/cli/aviator/grpc/GrpcUtil.java | 5 + .../cli/aviator/grpc/RequestWrapper.java | 2 + .../converter/SourceDecoderConverterTest.java | 41 +++ .../cli/aviator/audit/AuditFprStatusTest.java | 86 ++++++ .../fpr/utils/SourceCodeEnricherTest.java | 267 ++++++++++++++++++ .../aviator/fpr/utils/SourceDecodersTest.java | 126 +++++++++ .../cli/aviator/grpc/GrpcUtilTest.java | 15 + .../AviatorSSCApplyRemediationsCommand.java | 8 +- .../ssc/cli/cmd/AviatorSSCAuditCommand.java | 3 + .../ssc/helper/AviatorSSCAuditHelper.java | 29 +- .../aviator/i18n/AviatorMessages.properties | 3 +- .../cli/cmd/AviatorSSCAuditCommandTest.java | 48 ++++ .../FoDAviatorApplyRemediationsCommand.java | 5 +- .../cli/fod/i18n/FoDMessages.properties | 1 + 36 files changed, 1627 insertions(+), 215 deletions(-) create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverter.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/mixin/SourceEncodingsMixin.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CharsetSourceDecoder.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CompositeSourceDecoder.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FprSourceDecoder.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/ISourceDecoder.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncoder.java create mode 100644 fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverterTest.java create mode 100644 fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java create mode 100644 fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java create mode 100644 fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java diff --git a/fcli-core/fcli-app/build.gradle.kts b/fcli-core/fcli-app/build.gradle.kts index 12c38c2a221..3cbddfca67c 100644 --- a/fcli-core/fcli-app/build.gradle.kts +++ b/fcli-core/fcli-app/build.gradle.kts @@ -8,7 +8,7 @@ plugins { // Inter-project dependencies val refs = listOf( "fcliCommonRef","fcliCommonThirdpartyRef","fcliCommonCiRef","fcliCommonActionRef","fcliCommonToolRef", - "fcliActionRef","fcliAiAssistRef","fcliAviatorRef","fcliConfigRef", + "fcliActionRef","fcliAiAssistRef","fcliAviatorCommonRef","fcliAviatorRef","fcliConfigRef", "fcliFoDRef","fcliSSCRef","fcliSCSastRef","fcliSCDastRef", "fcliToolRef","fcliLicenseRef","fcliUtilRef" ) diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverter.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverter.java new file mode 100644 index 00000000000..083d74bbfe1 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverter.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator._common.cli.converter; + +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; + +import picocli.CommandLine.ITypeConverter; +import picocli.CommandLine.TypeConversionException; + +/** + * Picocli adapter: maps a single {@code --source-encodings} token to an + * {@link ISourceDecoder} via the domain factory {@link SourceDecoders}. + */ +public final class SourceDecoderConverter implements ITypeConverter { + @Override + public ISourceDecoder convert(String value) { + try { + return SourceDecoders.fromToken(value); + } catch (IllegalArgumentException e) { + // Covers blank tokens, IllegalCharsetNameException, UnsupportedCharsetException + throw new TypeConversionException( + e.getMessage() != null ? e.getMessage() : "Invalid source encoding '" + value + "'"); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/mixin/SourceEncodingsMixin.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/mixin/SourceEncodingsMixin.java new file mode 100644 index 00000000000..e58f0e86d0d --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/mixin/SourceEncodingsMixin.java @@ -0,0 +1,44 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator._common.cli.mixin; + +import java.util.List; + +import com.fortify.cli.aviator._common.cli.converter.SourceDecoderConverter; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; + +import lombok.Getter; +import picocli.CommandLine.Option; + +/** + * Shared {@code --source-encodings} option for Aviator commands that decode + * (and optionally re-encode) source files from an FPR. + */ +public class SourceEncodingsMixin { + @Getter + @Option(names = {"--source-encodings"}, + split = ",", + converter = SourceDecoderConverter.class, + defaultValue = SourceDecoders.DEFAULT_SOURCE_ENCODINGS, + paramLabel = "encoding", + descriptionKey = "fcli.aviator.source-encodings") + private List sourceDecoders; + + /** + * Returns a single decoder that tries the configured candidates in order. + */ + public ISourceDecoder getSourceDecoder() { + return SourceDecoders.of(sourceDecoders); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java index e2db9d777cb..23b77f0119a 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java @@ -12,6 +12,8 @@ */ package com.fortify.cli.aviator.applyRemediation; +import java.util.Objects; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,6 +22,8 @@ import com.fortify.cli.aviator.config.IAviatorLogger; import com.fortify.cli.aviator.fpr.processor.RemediationProcessor; import com.fortify.cli.aviator.fpr.processor.RemediationProcessor.RemediationMetric; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.util.FprHandle; @@ -28,6 +32,12 @@ public class ApplyAutoRemediationOnSource { public static RemediationMetric applyRemediations(FprHandle fprHandle, String sourceCodeDirectory, IAviatorLogger logger) throws AviatorSimpleException, AviatorTechnicalException { + return applyRemediations(fprHandle, sourceCodeDirectory, SourceDecoders.defaults(), logger); + } + + public static RemediationMetric applyRemediations(FprHandle fprHandle, String sourceCodeDirectory, + ISourceDecoder sourceDecoder, IAviatorLogger logger) + throws AviatorSimpleException, AviatorTechnicalException { LOG.info("Starting apply auto-remediation process for file: {}", fprHandle.getFprPath()); @@ -37,8 +47,8 @@ public static RemediationMetric applyRemediations(FprHandle fprHandle, String so } LOG.info("FPR validation successful"); - RemediationProcessor remediationProcessor = new RemediationProcessor(fprHandle, sourceCodeDirectory); + RemediationProcessor remediationProcessor = new RemediationProcessor(fprHandle, sourceCodeDirectory, + Objects.requireNonNull(sourceDecoder, "sourceDecoder")); return remediationProcessor.processRemediationXML(); - } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java index 4571af0bc7f..5888d18affc 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java @@ -14,6 +14,7 @@ import java.io.File; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -39,6 +40,7 @@ import com.fortify.cli.aviator.fpr.model.FPRInfo; import com.fortify.cli.aviator.fpr.processor.AuditProcessor; import com.fortify.cli.aviator.fpr.processor.StreamingFVDLProcessor; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.ResourceUtil; @@ -52,8 +54,11 @@ public static FPRAuditResult auditFPR(AuditFprOptions options) options.getFprHandle().validate(); AviatorConfigManager.getInstance(); + // Non-null: AuditFprOptions defaults via @Builder.Default; CLI mixin always supplies a decoder. + ISourceDecoder sourceDecoder = options.getSourceDecoder(); + // --- STAGE 1: PARSING --- - ParsedFprData parsedData = prepareAndParseFpr(options.getFprHandle()); + ParsedFprData parsedData = prepareAndParseFpr(options.getFprHandle(), sourceDecoder); TagMappingConfig tagMappingConfig = loadTagMappingConfig(options.getTagMappingPath()); Map issueCategoryLookup = tagMappingConfig.requiresCategoryForSuppressionEvaluation() ? buildIssueCategoryLookup(parsedData.vulnerabilities) @@ -69,22 +74,21 @@ public static FPRAuditResult auditFPR(AuditFprOptions options) Map auditResponses = new ConcurrentHashMap<>(); AuditOutcome auditOutcome = performAviatorAudit( parsedData, options.getLogger(), options.getToken(), options.getAppVersion(), options.getUrl(), options.getSscAppName(), options.getSscAppVersion(), - auditResponses, filterSelection, options.getFprHandle(), options.getFolderPriorityOrder() + auditResponses, filterSelection, options.getFprHandle(), options.getFolderPriorityOrder(), sourceDecoder ); // --- STAGE 4: FINALIZATION --- return finalizeFprAudit( auditOutcome, auditResponses, parsedData.auditProcessor, - tagMappingConfig, issueCategoryLookup, parsedData.fprInfo + tagMappingConfig, issueCategoryLookup, parsedData.fprInfo, parsedData.streamingFVDLProcessor ); } - private static ParsedFprData prepareAndParseFpr(FprHandle fprHandle) { + private static ParsedFprData prepareAndParseFpr(FprHandle fprHandle, ISourceDecoder sourceDecoder) { try { // Processors now take the FprHandle directly, no more extracted path - AuditProcessor auditProcessor = new AuditProcessor(fprHandle); - //FVDLProcessor fvdlProcessor = new FVDLProcessor(fprHandle); - StreamingFVDLProcessor streamingFVDLProcessor = new StreamingFVDLProcessor(fprHandle); + AuditProcessor auditProcessor = new AuditProcessor(fprHandle, sourceDecoder); + StreamingFVDLProcessor streamingFVDLProcessor = new StreamingFVDLProcessor(fprHandle, sourceDecoder); Map auditIssueMap = auditProcessor.processAuditXML(); FPRProcessor fprProcessor = new FPRProcessor(fprHandle, auditIssueMap, auditProcessor); @@ -126,7 +130,8 @@ private static Map buildIssueCategoryLookup(List private static AuditOutcome performAviatorAudit( ParsedFprData parsedData, IAviatorLogger logger, String token, String appVersion, String url, String sscAppName, String sscAppVersion, - Map auditResponsesToFill, FilterSelection filterSelection, FprHandle fprHandle, List folderPriorityOrder) { + Map auditResponsesToFill, FilterSelection filterSelection, FprHandle fprHandle, + List folderPriorityOrder, ISourceDecoder sourceDecoder) { SourceLanguageResolver sourceLanguageResolver = new SourceLanguageResolver(parsedData.streamingFVDLProcessor.getFvdlMetadata()); parsedData.streamingFVDLProcessor.getFvdlMetadata().clearSourceFileTypeIndexes(); @@ -141,7 +146,9 @@ private static AuditOutcome performAviatorAudit( filterSelection, logger, folderPriorityOrder, - sourceLanguageResolver + sourceLanguageResolver, + sourceDecoder, + parsedData.streamingFVDLProcessor.getFvdlMetadata() ); return issueAuditor.performAudit( auditResponsesToFill, token, appVersion, parsedData.fprInfo.getBuildId(), url, fprHandle @@ -151,33 +158,37 @@ private static AuditOutcome performAviatorAudit( private static FPRAuditResult finalizeFprAudit( AuditOutcome auditOutcome, Map auditResponses, AuditProcessor auditProcessor, TagMappingConfig tagMappingConfig, - Map issueCategoryLookup, FPRInfo fprInfo) { + Map issueCategoryLookup, FPRInfo fprInfo, StreamingFVDLProcessor streamingFVDLProcessor) { int totalIssuesToAudit = auditOutcome.getTotalIssuesToAudit(); + int issuesSubmitted = getSubmittedAuditCount(auditResponses); if (auditResponses.isEmpty()) { if (totalIssuesToAudit == 0) { LOG.info("No issues were audited, skipping update and upload"); - return new FPRAuditResult(null, "SKIPPED", "No issues to audit", 0, totalIssuesToAudit); + return new FPRAuditResult(null, "SKIPPED", "No issues to audit", 0, totalIssuesToAudit, + issuesSubmitted, 0, Map.of(), 0, Map.of()); } else { LOG.error("No audit responses received for {} issues", totalIssuesToAudit); - return new FPRAuditResult(null, "FAILED", "No audit responses received from server", 0, totalIssuesToAudit); + return new FPRAuditResult(null, "FAILED", "No audit responses received from server", 0, totalIssuesToAudit, + issuesSubmitted, 0, Map.of(), 0, Map.of()); } } long issuesSuccessfullyAudited = auditResponses.values().stream() .filter(response -> "SUCCESS".equalsIgnoreCase(response.getStatus())) .count(); + Map skippedByReason = getSkippedAuditReasons(auditResponses, totalIssuesToAudit); + int issuesSkipped = skippedByReason.values().stream().mapToInt(Integer::intValue).sum(); String status; String message = null; - if (issuesSuccessfullyAudited == totalIssuesToAudit) { - status = "AUDITED"; - } else if (issuesSuccessfullyAudited > 0) { - status = "PARTIALLY_AUDITED"; - } else { - status = "FAILED"; + status = determineAuditStatus(issuesSuccessfullyAudited, issuesSkipped, totalIssuesToAudit, auditResponses.size()); + if ("SKIPPED".equals(status)) { + message = String.format("All %d issues were skipped", totalIssuesToAudit); + } else if ("FAILED".equals(status)) { String commonFailureReason = auditResponses.values().stream() + .filter(response -> !"SKIPPED".equalsIgnoreCase(response.getStatus())) .map(AuditResponse::getStatusMessage) .filter(msg -> msg != null && !msg.isBlank()) .findFirst() @@ -186,16 +197,69 @@ private static FPRAuditResult finalizeFprAudit( if (commonFailureReason.startsWith("Client-side pre-processing error: ")) { commonFailureReason = commonFailureReason.substring("Client-side pre-processing error: ".length()); } - message = String.format("All %d issues failed (%s)", totalIssuesToAudit, commonFailureReason); + message = String.format("No issues were audited (%d skipped; failure details: %s)", + issuesSkipped, commonFailureReason); } File updatedFile = null; if (issuesSuccessfullyAudited > 0) { updatedFile = auditProcessor.updateAndSaveAuditAndRemediationsXml( - auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo); + auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo, + streamingFVDLProcessor.getFvdlMetadata()); + } + AuditProcessor.RemediationGenerationMetric remediationGenerationMetric = auditProcessor.getLastRemediationGenerationMetric(); + + if (!skippedByReason.isEmpty()) { + LOG.info("Skipped audit issues by reason: {}", skippedByReason); + } + if (!remediationGenerationMetric.skippedByReason().isEmpty()) { + LOG.info("Skipped audit remediation generation by reason: {}", remediationGenerationMetric.skippedByReason()); } LOG.info("FPR audit process completed with status: {}", status); - return new FPRAuditResult(updatedFile, status, message, (int) issuesSuccessfullyAudited, totalIssuesToAudit); + return new FPRAuditResult(updatedFile, status, message, (int) issuesSuccessfullyAudited, totalIssuesToAudit, + issuesSubmitted, issuesSkipped, skippedByReason, remediationGenerationMetric.skippedRemediations(), + remediationGenerationMetric.skippedByReason()); + } + + static int getSubmittedAuditCount(Map auditResponses) { + return (int) auditResponses.values().stream() + .filter(AuditResponse::isSubmittedToAviator) + .count(); + } + + static String determineAuditStatus(long issuesSuccessfullyAudited, int issuesSkipped, + int totalIssuesToAudit, int responseCount) { + if (issuesSuccessfullyAudited == totalIssuesToAudit) { + return "AUDITED"; + } + if (issuesSuccessfullyAudited > 0) { + return "PARTIALLY_AUDITED"; + } + if (issuesSkipped == totalIssuesToAudit && responseCount == totalIssuesToAudit) { + return "SKIPPED"; + } + return "FAILED"; + } + + static Map getSkippedAuditReasons(Map auditResponses, int totalIssuesToAudit) { + Map skippedByReason = new LinkedHashMap<>(); + auditResponses.values().stream() + .filter(response -> "SKIPPED".equalsIgnoreCase(response.getStatus())) + .map(AuditFPR::getSkippedAuditReason) + .forEach(reason -> recordSkipped(skippedByReason, reason)); + int missingResponses = Math.max(0, totalIssuesToAudit - auditResponses.size()); + if (missingResponses > 0) { + skippedByReason.merge("No audit response received", missingResponses, Integer::sum); + } + return skippedByReason; + } + + private static String getSkippedAuditReason(AuditResponse response) { + return response.getAuditSkipReason().getDisplayMessage(); + } + + private static void recordSkipped(Map skippedByReason, String reason) { + skippedByReason.merge(reason, 1, Integer::sum); } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java index d25c1286183..dc5b6897892 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java @@ -19,6 +19,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -48,7 +49,10 @@ import com.fortify.cli.aviator.fpr.filter.VulnerabilityFilterer; import com.fortify.cli.aviator.fpr.model.AuditIssue; import com.fortify.cli.aviator.fpr.model.FPRInfo; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.fpr.processor.AuditProcessor; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.grpc.AviatorGrpcClient; import com.fortify.cli.aviator.grpc.AviatorGrpcClientHelper; import com.fortify.cli.aviator.util.Constants; @@ -80,6 +84,8 @@ public class IssueAuditor { private TagDefinition humanAuditTag; private TagDefinition aviatorStatusTag; private final SourceLanguageResolver sourceLanguageResolver; + private final ISourceDecoder sourceDecoder; + private final FVDLMetadata fvdlMetadata; private final IAviatorLogger logger; private final List customPriorityOrder; @@ -88,6 +94,15 @@ public IssueAuditor(List vulnerabilities, AuditProcessor auditPro FPRInfo fprInfo, String SSCApplicationName, String SSCApplicationVersion, FilterSelection filterSelection, IAviatorLogger logger, List customPriorityOrder, SourceLanguageResolver sourceLanguageResolver) { + this(vulnerabilities, auditProcessor, auditIssueMap, fprInfo, SSCApplicationName, SSCApplicationVersion, + filterSelection, logger, customPriorityOrder, sourceLanguageResolver, SourceDecoders.defaults(), null); + } + + public IssueAuditor(List vulnerabilities, AuditProcessor auditProcessor, Map auditIssueMap, + FPRInfo fprInfo, String SSCApplicationName, String SSCApplicationVersion, + FilterSelection filterSelection, IAviatorLogger logger, List customPriorityOrder, + SourceLanguageResolver sourceLanguageResolver, ISourceDecoder sourceDecoder, + FVDLMetadata fvdlMetadata) { this.logger = logger; this.customPriorityOrder = customPriorityOrder; this.MAX_PER_CATEGORY = Constants.MAX_PER_CATEGORY; @@ -103,6 +118,8 @@ public IssueAuditor(List vulnerabilities, AuditProcessor auditPro this.SSCApplicationName = SSCApplicationName; this.SSCApplicationVersion = SSCApplicationVersion; this.sourceLanguageResolver = sourceLanguageResolver; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); + this.fvdlMetadata = fvdlMetadata; this.analysisTag = fprInfo.getFilterTemplate().getTagDefinitions().stream().filter(t -> "Analysis".equalsIgnoreCase(t.getName())).findFirst().orElse(null); this.resultsTag = resolveResultTag("", "", analysisTag); } @@ -162,7 +179,8 @@ public AuditOutcome performAudit(Map auditResponses, Stri } else { try (AviatorGrpcClient client = AviatorGrpcClientHelper.createClient(url, logger, DEFAULT_PING_INTERVAL_SECONDS)) { CompletableFuture> future = - client.processBatchRequests(promptsToAudit, projectName, fprInfo.getBuildId(), SSCApplicationName, SSCApplicationVersion, token, fprHandle, customPriorityOrder); + client.processBatchRequests(promptsToAudit, projectName, fprInfo.getBuildId(), SSCApplicationName, + SSCApplicationVersion, token, fprHandle, customPriorityOrder, sourceDecoder, fvdlMetadata); Map responses = future.get(500, TimeUnit.MINUTES); responses.forEach((requestId, response) -> auditResponses.put(response.getIssueId(), response)); logger.progress("Audit completed"); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java index 54b8c803038..8fa13198222 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java @@ -14,6 +14,8 @@ import java.util.List; import com.fortify.cli.aviator.config.IAviatorLogger; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.util.FprHandle; import lombok.Builder; @@ -34,4 +36,5 @@ public class AuditFprOptions { private final boolean noFilterSet; private final List folderNames; private final List folderPriorityOrder; + @Builder.Default private final ISourceDecoder sourceDecoder = SourceDecoders.defaults(); } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java index 220379cf776..8406c07cdde 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java @@ -12,32 +12,57 @@ */ package com.fortify.cli.aviator.audit.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.formkiq.graalvm.annotations.Reflectable; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; @Data -@AllArgsConstructor +@AllArgsConstructor(access = AccessLevel.PRIVATE) @NoArgsConstructor @Builder @Reflectable public class AuditResponse { + @Getter + @RequiredArgsConstructor + public enum AuditSkipReason { + SOURCE_FILE_DECODE_FAILED( + "Could not decode source file%s: %s%s", + "Source file decode failed"), + SOURCE_FILE_READ_FAILED( + "%s could not be read from the FPR%s", + "Source file read failed"), + SKIPPED_BY_AVIATOR("SKIPPED", "Skipped by Aviator"); + + private final String messageFormat; + private final String displayMessage; + + public String format(Object... args) { + return String.format(messageFormat, args); + } + } + private AuditResult auditResult; private int inputToken; private int outputToken; - private String status; private String statusMessage; + @JsonIgnore + private AuditSkipReason auditSkipReason; private String issueId; + @JsonIgnore + private boolean submittedToAviator; private String tier; private String aviatorPredictionTag; private Boolean isAviatorProcessed; private String userPrompt; private String systemPrompt; - } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java index 94d8b9a4e18..db7704cd69e 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java @@ -13,6 +13,7 @@ package com.fortify.cli.aviator.audit.model; import java.io.File; +import java.util.Map; import lombok.Data; @@ -23,13 +24,39 @@ public class FPRAuditResult { private String message; private int issuesSuccessfullyAudited; private int totalIssuesToAudit; + private int issuesSubmitted; + private int issuesSkipped; + private Map skippedByReason; + private int remediationGenerationSkipped; + private Map remediationGenerationSkippedByReason; public FPRAuditResult(File updatedFile, String status, String message, int issuesSuccessfullyAudited, int totalIssuesToAudit) { + this(updatedFile, status, message, issuesSuccessfullyAudited, totalIssuesToAudit, + totalIssuesToAudit, Math.max(0, totalIssuesToAudit - issuesSuccessfullyAudited), Map.of(), 0, Map.of()); + } + + public FPRAuditResult(File updatedFile, String status, String message, + int issuesSuccessfullyAudited, int totalIssuesToAudit, int issuesSkipped, + Map skippedByReason, int remediationGenerationSkipped, + Map remediationGenerationSkippedByReason) { + this(updatedFile, status, message, issuesSuccessfullyAudited, totalIssuesToAudit, totalIssuesToAudit, + issuesSkipped, skippedByReason, remediationGenerationSkipped, remediationGenerationSkippedByReason); + } + + public FPRAuditResult(File updatedFile, String status, String message, + int issuesSuccessfullyAudited, int totalIssuesToAudit, int issuesSubmitted, + int issuesSkipped, Map skippedByReason, int remediationGenerationSkipped, + Map remediationGenerationSkippedByReason) { this.updatedFile = updatedFile; this.status = status; this.message = message; this.issuesSuccessfullyAudited = issuesSuccessfullyAudited; this.totalIssuesToAudit = totalIssuesToAudit; + this.issuesSubmitted = issuesSubmitted; + this.issuesSkipped = issuesSkipped; + this.skippedByReason = skippedByReason == null ? Map.of() : Map.copyOf(skippedByReason); + this.remediationGenerationSkipped = remediationGenerationSkipped; + this.remediationGenerationSkippedByReason = remediationGenerationSkippedByReason == null ? Map.of() : Map.copyOf(remediationGenerationSkippedByReason); } } \ No newline at end of file diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java index 602367fbf03..d919ff321b6 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java @@ -27,8 +27,10 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -59,7 +61,10 @@ import com.fortify.cli.aviator.config.TagMappingConfig; import com.fortify.cli.aviator.fpr.model.AuditIssue; import com.fortify.cli.aviator.fpr.model.FPRInfo; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.fpr.utils.FileUtils; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; @@ -83,9 +88,39 @@ public class AuditProcessor { private final Map auditIssueMap = new HashMap<>(); private final FprHandle fprHandle; + private final ISourceDecoder sourceDecoder; + private RemediationGenerationMetric lastRemediationGenerationMetric = RemediationGenerationMetric.empty(); + + public record RemediationGenerationMetric(int skippedRemediations, Map skippedByReason) { + public static RemediationGenerationMetric empty() { + return new RemediationGenerationMetric(0, Map.of()); + } + } + + private enum RemediationSkipReason { + SOURCE_READ_OR_DECODE_FAILED("Source file read/decode failed"), + INVALID_LINE_NUMBER("Invalid line number"), + STRUCTURALLY_INVALID("Structurally invalid remediation"), + NO_VALID_CHANGES("No valid changes"); + + private final String label; + + RemediationSkipReason(String label) { + this.label = label; + } + } public AuditProcessor(FprHandle fprHandle) { + this(fprHandle, SourceDecoders.defaults()); + } + + public AuditProcessor(FprHandle fprHandle, ISourceDecoder sourceDecoder) { this.fprHandle = fprHandle; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); + } + + public RemediationGenerationMetric getLastRemediationGenerationMetric() { + return lastRemediationGenerationMetric; } /** @@ -709,6 +744,13 @@ private String addCommentToIssueElement(Element issueElement, String commentText public File updateAndSaveAuditAndRemediationsXml(Map auditResponses, TagMappingConfig tagMappingConfig, Map issueCategoryLookup, FPRInfo fprInfo) throws AviatorTechnicalException { + return updateAndSaveAuditAndRemediationsXml(auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo, null); + } + + public File updateAndSaveAuditAndRemediationsXml(Map auditResponses, + TagMappingConfig tagMappingConfig, Map issueCategoryLookup, + FPRInfo fprInfo, FVDLMetadata fvdlMetadata) throws AviatorTechnicalException { + lastRemediationGenerationMetric = RemediationGenerationMetric.empty(); // Step 1: Apply this save's audit responses. writtenInstanceIds is the local retain set. Map effectiveIssueCategoryLookup = issueCategoryLookup == null ? Map.of() : issueCategoryLookup; AuditXmlUpdateResult updateResult = updateAuditXml( @@ -728,7 +770,10 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi // Step 4: Generate the in-memory remediations.xml document if needed. if (hasRemediations && !remediationCommentTimestamps.isEmpty()) { - this.remediationsDoc = generateRemediationsXml(auditResponses, remediationCommentTimestamps, fprInfo); + Map skippedByReason = new LinkedHashMap<>(); + this.remediationsDoc = generateRemediationsXml(auditResponses, remediationCommentTimestamps, fprInfo, + fvdlMetadata, skippedByReason); + lastRemediationGenerationMetric = toRemediationGenerationMetric(skippedByReason); } else { this.remediationsDoc = null; if (hasRemediations) { @@ -765,7 +810,8 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi private Document generateRemediationsXml(Map auditResponses, Map remediationCommentTimestamps, - FPRInfo fprInfo) throws AviatorTechnicalException { + FPRInfo fprInfo, FVDLMetadata fvdlMetadata, + Map skippedByReason) throws AviatorTechnicalException { try { DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); docFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); @@ -794,6 +840,7 @@ private Document generateRemediationsXml(Map auditRespons Element remediationListElement = finalDoc.createElementNS(REMEDIATIONS_NAMESPACE_URI, "RemediationList"); rootElement.appendChild(remediationListElement); + FileUtils fileUtils = new FileUtils(this.sourceDecoder, fvdlMetadata); int validRemediationCount = 0; for (Map.Entry entry : auditResponses.entrySet()) { @@ -828,12 +875,11 @@ private Document generateRemediationsXml(Map auditRespons filenameElement.setTextContent(filename); fileChangesElement.appendChild(filenameElement); - //Optional originalFileContentOptional = fvdlProcessor.getSourceFileContent(filename); - FileUtils fileUtils = new FileUtils(); - Optional originalFileContentOptional = fileUtils.getSourceFileContent(fprHandle, filename); + Optional originalFileContentOptional = fileUtils.getSourceFileContent(fprHandle, filename); if (originalFileContentOptional.isEmpty()) { logger.warn("WARN: Could not retrieve source code for file '{}'. Skipping remediation generation for this file for instanceId '{}'.", filename, instanceId); + recordSkipped(skippedByReason, RemediationSkipReason.SOURCE_READ_OR_DECODE_FAILED); continue; } @@ -900,6 +946,7 @@ private Document generateRemediationsXml(Map auditRespons fileChangesElement.appendChild(changeElement); } catch (NumberFormatException e) { logger.error("Skipping change for issue {} due to invalid line number format. Details: {}", instanceId, e.getMessage()); + recordSkipped(skippedByReason, RemediationSkipReason.INVALID_LINE_NUMBER); } } if (fileChangesElement.getElementsByTagNameNS(REMEDIATIONS_NAMESPACE_URI, "Change").getLength() > 0) { @@ -915,9 +962,11 @@ private Document generateRemediationsXml(Map auditRespons validRemediationCount++; } else { logger.warn("WARN: Skipping structurally invalid remediation for issue instanceId: {}", instanceId); + recordSkipped(skippedByReason, RemediationSkipReason.STRUCTURALLY_INVALID); } } else { logger.warn("WARN: Skipping remediation for instanceId '{}' because all of its proposed changes were invalid and could not be processed.", instanceId); + recordSkipped(skippedByReason, RemediationSkipReason.NO_VALID_CHANGES); } } @@ -929,6 +978,15 @@ private Document generateRemediationsXml(Map auditRespons } } + private RemediationGenerationMetric toRemediationGenerationMetric(Map skippedByReason) { + int skippedRemediations = skippedByReason.values().stream().mapToInt(Integer::intValue).sum(); + return new RemediationGenerationMetric(skippedRemediations, Map.copyOf(skippedByReason)); + } + + private void recordSkipped(Map skippedByReason, RemediationSkipReason reason) { + skippedByReason.merge(reason.label, 1, Integer::sum); + } + private String calculateHashBase64(String content, String algorithm) { if (content == null) return ""; try { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java index 97c1ffce863..2a059b8380b 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java @@ -14,11 +14,7 @@ import java.io.IOException; import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; -import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -32,6 +28,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.zip.ZipFile; @@ -49,6 +46,12 @@ import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.DecodeResult; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.SourceDecodeException; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder.SourceEncodeException; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.FuzzyContextSearcher; @@ -58,6 +61,7 @@ public class RemediationProcessor { private final FprHandle fprHandle; private final String sourceCodeDirectory; + private final ISourceDecoder sourceDecoder; public record RemediationMetric(int totalRemediations, int appliedRemediations, int skippedRemediations, Set modifiedFiles, Map skippedByReason) { @@ -66,16 +70,14 @@ public RemediationMetric(int totalRemediations, int appliedRemediations, int ski } } - private record FvdlMetadataResult(FVDLMetadata metadata, SkipReason skipReason) {} + private record SourceFileContent(String content, Charset charset, String encodingSource) {} - private record PendingFileWrite(String filename, Path filePath, String content, byte[] updatedBytes) {} + private record PendingFileWrite(String filename, Path filePath, String content, Charset charset, String encodingSource, + byte[] updatedBytes) {} private record RollbackFileWrite(String filename, Path filePath, byte[] originalBytes) {} private enum SkipReason { - FVDL_METADATA_UNAVAILABLE("FVDL metadata unavailable"), - FVDL_ENCODING_MISSING("FVDL source encoding missing"), - FVDL_ENCODING_UNSUPPORTED("FVDL source encoding unsupported"), SOURCE_FILE_MISSING("Source file missing"), SOURCE_FILE_OUTSIDE_SOURCE_DIR("Source file outside source directory"), SOURCE_READ_FAILED("Source file read failed"), @@ -136,8 +138,13 @@ private static class RollbackRemediationException extends AviatorTechnicalExcept } public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory) { + this(fprHandle, sourceCodeDirectory, SourceDecoders.defaults()); + } + + public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory, ISourceDecoder sourceDecoder) { this.fprHandle = fprHandle; this.sourceCodeDirectory = sourceCodeDirectory; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); } public RemediationMetric processRemediationXML() { @@ -157,7 +164,7 @@ public RemediationMetric processRemediationXML() { } final Path sourceBasePath = Paths.get(trimmedSourceDir).toAbsolutePath().normalize(); LOG.debug("Applying remediations from {} to source directory {}", remediationPath, sourceBasePath); - final FvdlMetadataResult fvdlMetadataResult = loadFvdlMetadata(); + final FVDLMetadata fvdlMetadata = loadFvdlMetadata(); try (InputStream remediationStream = Files.newInputStream(remediationPath)) { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); @@ -176,7 +183,7 @@ public RemediationMetric processRemediationXML() { appliedRemediations = 0; for (int i = 0; i < remediationNodes.getLength(); i++) { Element remediation = (Element) remediationNodes.item(i); - if (processRemediation(remediation, sourceBasePath, fvdlMetadataResult, modifiedFiles, skippedByReason)) { + if (processRemediation(remediation, sourceBasePath, fvdlMetadata, modifiedFiles, skippedByReason)) { appliedRemediations++; } } @@ -198,11 +205,11 @@ public RemediationMetric processRemediationXML() { return new RemediationMetric(totalRemediations, appliedRemediations, skippedRemediations, modifiedFiles, skippedByReason); } - private boolean processRemediation(Element remediation, Path sourceBasePath, FvdlMetadataResult fvdlMetadataResult, + private boolean processRemediation(Element remediation, Path sourceBasePath, FVDLMetadata fvdlMetadata, Set modifiedFiles, Map skippedByReason) { String instanceId = remediation.getAttribute("instanceId"); try { - Map pendingWrites = prepareFileChanges(remediation, sourceBasePath, fvdlMetadataResult); + Map pendingWrites = prepareFileChanges(remediation, sourceBasePath, fvdlMetadata); if (pendingWrites.isEmpty()) { recordSkipped(skippedByReason, SkipReason.NO_CHANGES); return false; @@ -230,7 +237,7 @@ private boolean processRemediation(Element remediation, Path sourceBasePath, Fvd } private Map prepareFileChanges(Element remediation, Path sourceBasePath, - FvdlMetadataResult fvdlMetadataResult) { + FVDLMetadata fvdlMetadata) { NodeList fileChangesNodes = remediation.getElementsByTagNameNS(NAMESPACE_URI, "FileChanges"); if (fileChangesNodes.getLength() == 0) { throw new SkipRemediationException(SkipReason.NO_CHANGES, "No file changes found"); @@ -238,12 +245,12 @@ private Map prepareFileChanges(Element remediation, Path Map pendingWrites = new LinkedHashMap<>(); for (int j = 0; j < fileChangesNodes.getLength(); j++) { - processFileChanges(remediation, (Element) fileChangesNodes.item(j), sourceBasePath, fvdlMetadataResult, pendingWrites); + processFileChanges(remediation, (Element) fileChangesNodes.item(j), sourceBasePath, fvdlMetadata, pendingWrites); } return pendingWrites; } - private boolean processFileChanges(Element remediation, Element fileChanges, Path sourceBasePath, FvdlMetadataResult fvdlMetadataResult, + private boolean processFileChanges(Element remediation, Element fileChanges, Path sourceBasePath, FVDLMetadata fvdlMetadata, Map pendingWrites) { String instanceId = remediation.getAttribute("instanceId"); String filename = getRequiredElementText(fileChanges, "Filename"); @@ -260,23 +267,25 @@ private boolean processFileChanges(Element remediation, Element fileChanges, Pat } String fileHash = getRequiredElementText(fileChanges, "Hash"); - Charset sourceEncoding = getRequiredSourceEncoding(filename, fvdlMetadataResult); NodeList changesNodes = fileChanges.getElementsByTagNameNS(NAMESPACE_URI, "Change"); if (changesNodes.getLength() == 0) { throw new SkipRemediationException(SkipReason.NO_CHANGES, "No changes found for file: " + filename); } - LOG.debug("Remediation {} has {} change(s) for '{}' using FVDL encoding {}", instanceId, changesNodes.getLength(), filename, - sourceEncoding.name()); + SourceFileContent sourceFileContent = getPendingOrSourceContent(filePath, filename, fvdlMetadata, pendingWrites); + Charset sourceEncoding = sourceFileContent.charset(); + LOG.debug("Remediation {} has {} change(s) for '{}' using source encoding {}", instanceId, changesNodes.getLength(), filename, + sourceFileContent.encodingSource()); - String updatedContent = getPendingOrSourceContent(filePath, filename, sourceEncoding, pendingWrites); + String updatedContent = sourceFileContent.content(); for (int k = 0; k < changesNodes.getLength(); k++) { updatedContent = applyChange(instanceId, filename, fileHash, sourceEncoding, updatedContent, (Element) changesNodes.item(k), k + 1); } - byte[] updatedBytes = encodeStrict(updatedContent, sourceEncoding, filename); - pendingWrites.put(filePath, new PendingFileWrite(filename, filePath, updatedContent, updatedBytes)); - LOG.debug("Staged remediation {} for '{}' using FVDL encoding {}; changes={}, encodedBytes={}", instanceId, filename, - sourceEncoding.name(), changesNodes.getLength(), updatedBytes.length); + byte[] updatedBytes = encodeSourceFile(updatedContent, sourceEncoding, filename); + pendingWrites.put(filePath, new PendingFileWrite(filename, filePath, updatedContent, sourceEncoding, + sourceFileContent.encodingSource(), updatedBytes)); + LOG.debug("Staged remediation {} for '{}' using source encoding {}; changes={}, encodedBytes={}", instanceId, filename, + sourceFileContent.encodingSource(), changesNodes.getLength(), updatedBytes.length); return true; } @@ -334,10 +343,12 @@ private String applyChange(String instanceId, String filename, String fileHash, return String.join(lineSeparator, updatedLines); } - private String getPendingOrSourceContent(Path filePath, String filename, Charset sourceEncoding, + private SourceFileContent getPendingOrSourceContent(Path filePath, String filename, FVDLMetadata fvdlMetadata, Map pendingWrites) { PendingFileWrite pendingWrite = pendingWrites.get(filePath); - return pendingWrite == null ? readSourceFile(filePath, filename, sourceEncoding) : pendingWrite.content(); + return pendingWrite == null + ? readSourceFile(filePath, filename, fvdlMetadata) + : new SourceFileContent(pendingWrite.content(), pendingWrite.charset(), pendingWrite.encodingSource()); } private void commitRemediationWrites(String instanceId, Map pendingWrites, Set modifiedFiles) @@ -392,84 +403,46 @@ private boolean isFilePresent(Path path) { return Files.exists(path) && Files.isRegularFile(path); } - private FvdlMetadataResult loadFvdlMetadata() { + /** Nullable: missing/unreadable FVDL means FPR encoding candidate is skipped. */ + private FVDLMetadata loadFvdlMetadata() { if (!Files.exists(fprHandle.getPath("/audit.fvdl"))) { - LOG.warn("FVDL file '/audit.fvdl' is missing; source remediations will be skipped"); - return new FvdlMetadataResult(null, SkipReason.FVDL_METADATA_UNAVAILABLE); + LOG.warn("FVDL file '/audit.fvdl' is missing; FPR encoding candidate will be skipped"); + return null; } try (ZipFile zipFile = new ZipFile(fprHandle.getFprPath().toFile())) { LOG.debug("Loading FVDL build metadata from '{}' to resolve source encodings", fprHandle.getFprPath()); - StreamingFVDLProcessor processor = new StreamingFVDLProcessor(fprHandle); + // Decoder unused for metadata-only parse; ctor requires one for FileUtils wiring. + StreamingFVDLProcessor processor = new StreamingFVDLProcessor(fprHandle, sourceDecoder); processor.parseBuildMetadata(zipFile, "audit.fvdl"); LOG.debug("Loaded FVDL build metadata from '{}'", fprHandle.getFprPath()); - return new FvdlMetadataResult(processor.getFvdlMetadata(), null); + return processor.getFvdlMetadata(); } catch (Exception e) { - LOG.warn("Error reading source file encodings from audit.fvdl; source remediations will be skipped", e); - return new FvdlMetadataResult(null, SkipReason.FVDL_METADATA_UNAVAILABLE); + LOG.warn("Error reading source file encodings from audit.fvdl; FPR encoding candidate will be skipped", e); + return null; } } - private Charset getRequiredSourceEncoding(String filename, FvdlMetadataResult fvdlMetadataResult) { - if (fvdlMetadataResult.skipReason() != null || fvdlMetadataResult.metadata() == null) { - throw new SkipRemediationException(SkipReason.FVDL_METADATA_UNAVAILABLE, - "FVDL metadata is unavailable; cannot determine source encoding for file '" + filename + "'"); - } - - String encoding = fvdlMetadataResult.metadata().findSourceFileEncodingForFileName(filename); - if (encoding == null || encoding.isBlank()) { - LOG.debug("FVDL source encoding lookup failed for '{}'", filename); - throw new SkipRemediationException(SkipReason.FVDL_ENCODING_MISSING, - "FVDL does not declare a source encoding for file '" + filename + "'"); - } - - try { - Charset charset = Charset.forName(encoding); - LOG.debug("FVDL source encoding for '{}' resolved to '{}'", filename, charset.name()); - return charset; - } catch (Exception e) { - throw new SkipRemediationException(SkipReason.FVDL_ENCODING_UNSUPPORTED, - "FVDL declares unsupported source encoding '" + encoding + "' for file '" + filename + "'", e); - } - } - - private String readSourceFile(Path filePath, String filename, Charset sourceEncoding) { + private SourceFileContent readSourceFile(Path filePath, String filename, FVDLMetadata fvdlMetadata) { try { byte[] sourceBytes = Files.readAllBytes(filePath); - String decodedContent = decodeStrict(sourceBytes, sourceEncoding); - LOG.debug("Strict decoded '{}' using {}; sourceBytes={}, decodedChars={}", filename, sourceEncoding.name(), sourceBytes.length, - decodedContent.length()); - return decodedContent; - } catch (CharacterCodingException e) { - throw new SkipRemediationException(SkipReason.SOURCE_DECODE_FAILED, - "FVDL declares source encoding '" + sourceEncoding.name() + "' for file '" + filename + - "', but the source file cannot be decoded using that encoding", e); + // Metadata may be null (FVDL missing); FPR candidate fails and other encodings are tried. + DecodeResult decodeResult = sourceDecoder.decode(sourceBytes, filename, fvdlMetadata); + LOG.debug("Strict decoded '{}' using {}; sourceBytes={}, decodedChars={}", filename, decodeResult.source(), sourceBytes.length, + decodeResult.content().length()); + return new SourceFileContent(decodeResult.content(), decodeResult.charset(), decodeResult.source()); + } catch (SourceDecodeException e) { + throw new SkipRemediationException(SkipReason.SOURCE_DECODE_FAILED, e.getMessage(), e); } catch (IOException e) { throw new SkipRemediationException(SkipReason.SOURCE_READ_FAILED, "Error reading source code file '" + filePath + "'", e); } } - private String decodeStrict(byte[] bytes, Charset charset) throws CharacterCodingException { - return charset.newDecoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - .decode(ByteBuffer.wrap(bytes)) - .toString(); - } - - private byte[] encodeStrict(String content, Charset charset, String filename) { + private byte[] encodeSourceFile(String content, Charset charset, String filename) { try { - ByteBuffer buffer = charset.newEncoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - .encode(CharBuffer.wrap(content)); - byte[] result = new byte[buffer.remaining()]; - buffer.get(result); - return result; - } catch (CharacterCodingException e) { - throw new SkipRemediationException(SkipReason.REMEDIATION_ENCODE_FAILED, - "Remediation content for file '" + filename + "' cannot be encoded using FVDL source encoding '" + - charset.name() + "'", e); + return SourceEncoder.encode(content, charset, filename); + } catch (SourceEncodeException e) { + throw new SkipRemediationException(SkipReason.REMEDIATION_ENCODE_FAILED, e.getMessage(), e); } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java index 9a3f8ffa9e8..3dace7f6e75 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java @@ -32,6 +32,8 @@ import com.fortify.cli.aviator.fpr.filter.AnalyzerType; import com.fortify.cli.aviator.fpr.model.*; import com.fortify.cli.aviator.fpr.utils.FileUtils; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.fpr.utils.XmlUtils; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.StringUtil; @@ -70,17 +72,26 @@ public class StreamingFVDLProcessor { private long peakMemoryPass2 = 0; private long peakMemoryPostProcessing = 0; - public StreamingFVDLProcessor(FprHandle fprHandle){ + public StreamingFVDLProcessor(FprHandle fprHandle) { + this(fprHandle, SourceDecoders.defaults()); + } + + /** + * @param sourceDecoder used for stack-trace line/fragment source reads; shares {@link #fvdlMetadata} + * so FPR encoding candidates resolve after build metadata is parsed. + */ + public StreamingFVDLProcessor(FprHandle fprHandle, ISourceDecoder sourceDecoder) { this.vulnFinalizer = new VulnFinalizer(); - this.fileUtils = new FileUtils(); this.fprHandle = fprHandle; this.sourceFileMap = fprHandle.getSourceFileMap(); this.xmlInputFactory = XMLInputFactory.newInstance(); // Security: Disable external entity processing xmlInputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); - //this.parsingMetadata = new ParsingMetadata(); this.fvdlMetadata = new FVDLMetadata(); + // Same metadata instance FileUtils will see once encodings are registered during parse. + this.fileUtils = new FileUtils( + Objects.requireNonNull(sourceDecoder, "sourceDecoder"), this.fvdlMetadata); this.rawVulnerabilities = new ArrayList<>(); this.vulnerabilities = new ArrayList<>(); this.descriptionProcessor = new DescriptionProcessor(); @@ -92,8 +103,6 @@ public StreamingFVDLProcessor(FprHandle fprHandle){ this.traceParser.setNodeParser(nodeParser); // Circular dependency for Reason parsing this.descriptionParser = new DescriptionParser(); this.metadataParser = new MetadataParser(); - /*this.extractedPath = extractedPath; - this.indexXMLProcessor = new IndexXMLProcessor(extractedPath, sourceFileMap);*/ } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CharsetSourceDecoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CharsetSourceDecoder.java new file mode 100644 index 00000000000..b0cf5d06621 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CharsetSourceDecoder.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.util.Objects; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +/** Fixed-{@link Charset} decoder. Created via {@link SourceDecoders}. */ +final class CharsetSourceDecoder implements ISourceDecoder { + private final Charset charset; + private final String label; + + CharsetSourceDecoder(String charsetName) { + this.charset = Charset.forName(charsetName); + this.label = charsetName; + } + + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata) { + Objects.requireNonNull(bytes, "bytes must not be null"); + try { + return new DecodeResult(SourceEncoder.decodeStrict(bytes, charset), charset, label); + } catch (CharacterCodingException e) { + throw new SourceDecodeException(label + " failed to decode source bytes", e); + } + } + + @Override + public String describe() { + return label; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CompositeSourceDecoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CompositeSourceDecoder.java new file mode 100644 index 00000000000..9c335cb550f --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CompositeSourceDecoder.java @@ -0,0 +1,53 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +/** Tries candidates in order. Created via {@link SourceDecoders#of}. */ +final class CompositeSourceDecoder implements ISourceDecoder { + private final List decoders; + + CompositeSourceDecoder(List decoders) { + Objects.requireNonNull(decoders, "decoders must not be null"); + if (decoders.isEmpty()) { + throw new IllegalArgumentException("decoders must not be empty"); + } + this.decoders = List.copyOf(decoders); + } + + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata) { + Objects.requireNonNull(bytes, "bytes must not be null"); + List failures = new ArrayList<>(); + for (ISourceDecoder decoder : decoders) { + try { + return decoder.decode(bytes, filename, fvdlMetadata); + } catch (SourceDecodeException e) { + failures.add(e.getMessage() != null ? e.getMessage() : decoder.describe() + " failed"); + } + } + throw new SourceDecodeException("Could not decode source file '" + filename + "' using source encodings " + + describe() + "; attempted: " + String.join("; ", failures)); + } + + @Override + public String describe() { + return decoders.stream().map(ISourceDecoder::describe).collect(Collectors.joining(",")); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java index 6ecd49aed6c..f5878cc62d8 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java @@ -14,13 +14,13 @@ import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -28,15 +28,31 @@ import org.slf4j.LoggerFactory; import com.fortify.cli.aviator.audit.model.Fragment; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.util.FileTypeLanguageMapperUtil; import com.fortify.cli.aviator.util.FileUtil; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.LanguageCommentMapperUtil; import com.fortify.cli.aviator.util.StringUtil; +/** + * Source file helpers. Decode failures are soft (empty result + log): snippets/lines are best-effort. + * Callers that must fail or skip with metrics (e.g. remediation apply) should decode themselves. + */ public class FileUtils { private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); private final Map> fileContentCache = new ConcurrentHashMap<>(); + private final ISourceDecoder sourceDecoder; + private final FVDLMetadata fvdlMetadata; + + public FileUtils() { + this(SourceDecoders.defaults(), null); + } + + public FileUtils(ISourceDecoder sourceDecoder, FVDLMetadata fvdlMetadata) { + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); + this.fvdlMetadata = fvdlMetadata; + } /** * Reads all lines from a file, caching the result to avoid repeated reads. @@ -46,13 +62,17 @@ public class FileUtils { * @return List of lines, or empty list if file not found or error occurs */ public List readFileWithFallback(Path filePath) { + return readFileWithFallback(filePath, filePath.getFileName().toString()); + } + + private List readFileWithFallback(Path filePath, String filename) { return fileContentCache.computeIfAbsent(filePath, path -> { try { byte[] fileBytes = Files.readAllBytes(path); - String content = new String(fileBytes, StandardCharsets.UTF_8); + String content = sourceDecoder.decode(fileBytes, filename, fvdlMetadata).content(); return Arrays.asList(content.split("\\r?\\n")); - } catch (IOException e) { - logger.error("Failed to read file: {}", path, e); + } catch (IOException | ISourceDecoder.SourceDecodeException e) { + logger.warn("Could not read or decode source file {}: {}", path, e.getMessage()); return Collections.emptyList(); } }); @@ -80,7 +100,7 @@ public String getLineFromFile(FprHandle fprHandle, String relativePath, int line Path fullSourcePath = resolveFullPath(fprHandle, relativePath); if (fullSourcePath == null) return ""; - List lines = readFileWithFallback(fullSourcePath); + List lines = readFileWithFallback(fullSourcePath, relativePath); if (lineNumber > 0 && lines.size() >= lineNumber) { return appendLineNumbers(lines.get(lineNumber - 1), relativePath, lineNumber - 1); } @@ -97,7 +117,7 @@ public Fragment getFragmentFromFile(FprHandle fprHandle, String relativePath, in return new Fragment("", 0, 0); } - List lines = readFileWithFallback(fullSourcePath); + List lines = readFileWithFallback(fullSourcePath, relativePath); if (lines.isEmpty() || lineNumber <= 0) { return new Fragment("", 0, 0); } @@ -127,20 +147,24 @@ private Path resolveFullPath(FprHandle fprHandle, String relativePath) { return fprHandle.getPath(internalPath); } - - // Assumes you have already updated the signature to accept extractedPath public Optional getSourceFileContent(FprHandle fprHandle, String relativePath) { - Path actualSourcePath = resolveFullPath(fprHandle, relativePath); - if (actualSourcePath == null) { + try { + return Optional.of(readSourceFileContentStrict(fprHandle, relativePath)); + } catch (IOException | ISourceDecoder.SourceDecodeException e) { + logger.warn("Could not read source file content for path {}: {}", relativePath, e.getMessage()); return Optional.empty(); } + } - try { - return Optional.of(String.join(System.lineSeparator(), readFileWithFallback(actualSourcePath))); - } catch (Exception e) { - logger.warn("Could not read source file content for path: {}", relativePath, e); - return Optional.empty(); + /** Reads and decodes source content while preserving read or decode failures for the caller. */ + String readSourceFileContentStrict(FprHandle fprHandle, String relativePath) throws IOException { + Path actualSourcePath = resolveFullPath(fprHandle, relativePath); + if (actualSourcePath == null) { + throw new IOException("Source file key not found in sourceFileMap: " + relativePath); } + + byte[] fileBytes = Files.readAllBytes(actualSourcePath); + return sourceDecoder.decode(fileBytes, relativePath, fvdlMetadata).content(); } public String appendLineNumbers(String content, String fileName, int startLineNo) { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FprSourceDecoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FprSourceDecoder.java new file mode 100644 index 00000000000..f7241d3ed73 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FprSourceDecoder.java @@ -0,0 +1,57 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.util.Objects; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +/** Resolves encoding from FVDL metadata, then strict-decodes. Created via {@link SourceDecoders}. */ +final class FprSourceDecoder implements ISourceDecoder { + static final String TOKEN = "FPR"; + + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata) { + Objects.requireNonNull(bytes, "bytes must not be null"); + Charset charset = resolveCharset(filename, fvdlMetadata); + try { + String content = SourceEncoder.decodeStrict(bytes, charset); + return new DecodeResult(content, charset, TOKEN + "(" + charset.name() + ")"); + } catch (CharacterCodingException e) { + throw new SourceDecodeException(TOKEN + "(" + charset.name() + ") failed to decode source bytes", e); + } + } + + private static Charset resolveCharset(String filename, FVDLMetadata fvdlMetadata) { + if (fvdlMetadata == null) { + throw new SourceDecodeException("FPR metadata unavailable"); + } + String encodingName = fvdlMetadata.findSourceFileEncodingForFileName(filename); + if (encodingName == null || encodingName.isBlank()) { + throw new SourceDecodeException("FPR encoding missing for '" + filename + "'"); + } + try { + return Charset.forName(encodingName); + } catch (Exception e) { + throw new SourceDecodeException( + TOKEN + " resolved to unsupported encoding '" + encodingName + "'", e); + } + } + + @Override + public String describe() { + return TOKEN; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/ISourceDecoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/ISourceDecoder.java new file mode 100644 index 00000000000..96e76154a8b --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/ISourceDecoder.java @@ -0,0 +1,55 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.charset.Charset; + +import com.fortify.cli.aviator._common.exception.AviatorSimpleException; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +/** + * Decodes source file bytes using a specific encoding strategy. + * Implementations may resolve the encoding from FVDL metadata, use a fixed + * {@link Charset}, or try multiple candidate decoders in order. + */ +public interface ISourceDecoder { + /** + * Decode source file bytes into text. + * + * @param bytes raw source file bytes + * @param filename source file name (used for FPR encoding lookup and error messages) + * @param fvdlMetadata optional FVDL metadata; required only by FPR-based decoders + * @return decoded content together with the charset that produced it + * @throws SourceDecodeException if this decoder cannot decode the bytes + */ + DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata); + + /** + * Human-readable description of this decoder (used in CLI help and error messages). + */ + String describe(); + + record DecodeResult(String content, Charset charset, String source) {} + + class SourceDecodeException extends AviatorSimpleException { + private static final long serialVersionUID = 1L; + + public SourceDecodeException(String message) { + super(message); + } + + public SourceDecodeException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java index 57e28043765..873a5a82fec 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java @@ -13,17 +13,21 @@ package com.fortify.cli.aviator.fpr.utils; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.File; import com.fortify.cli.aviator.audit.model.StackTraceElement; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.StringUtil; @@ -39,24 +43,17 @@ public class SourceCodeEnricher { private static final Logger logger = LoggerFactory.getLogger(SourceCodeEnricher.class); - /*private final Path extractedPath; - private final Map sourceFileMap;*/ private final FprHandle fprHandle; private final FileUtils fileUtils; + private final Map sourceFileCache = new ConcurrentHashMap<>(); - /** - * Creates a new SourceCodeEnricher with the required dependencies. - * @param fprHandle Utility for file operations (line numbering, line counting) - */ - /*public SourceCodeEnricher(Path extractedPath, Map sourceFileMap, FileUtils fileUtils) { - this.extractedPath = extractedPath; - this.sourceFileMap = sourceFileMap; - this.fileUtils = fileUtils; - }*/ + public SourceCodeEnricher(FprHandle fprHandle) { + this(fprHandle, SourceDecoders.defaults(), null); + } - public SourceCodeEnricher(FprHandle fprHandle){ + public SourceCodeEnricher(FprHandle fprHandle, ISourceDecoder sourceDecoder, FVDLMetadata fvdlMetadata) { this.fprHandle = fprHandle; - this.fileUtils = new FileUtils(); + this.fileUtils = new FileUtils(Objects.requireNonNull(sourceDecoder, "sourceDecoder"), fvdlMetadata); } /** @@ -74,36 +71,41 @@ public SourceCodeEnricher(FprHandle fprHandle){ * @return Map of filename → File objects with content loaded */ public Map enrichWithSourceCode(List> stackTraces) { - Map uniqueFiles = new HashMap<>(); + return new HashMap<>(enrichWithSourceCodeDetailed(stackTraces).files()); + } + + public EnrichmentResult enrichWithSourceCodeDetailed(List> stackTraces) { + Map uniqueFiles = new LinkedHashMap<>(); + Map failuresByFilename = new LinkedHashMap<>(); if (stackTraces == null || stackTraces.isEmpty()) { logger.debug("No stack traces to enrich"); - return uniqueFiles; + return new EnrichmentResult(uniqueFiles, List.of()); } - processStackTraces(stackTraces, uniqueFiles); + processStackTraces(stackTraces, uniqueFiles, failuresByFilename); logger.debug("Enriched {} unique source files from {} stack traces", uniqueFiles.size(), stackTraces.size()); - return uniqueFiles; + return new EnrichmentResult(uniqueFiles, new ArrayList<>(failuresByFilename.values())); } /** * Processes all stack traces to extract and load unique source files. * Replicates FVDLProcessor.processStackTraceElements() logic. */ - private void processStackTraces(List> stackTraces, Map uniqueFiles) { + private void processStackTraces(List> stackTraces, Map uniqueFiles, + Map failuresByFilename) { for (List stackTrace : stackTraces) { if (stackTrace == null) continue; for (StackTraceElement element : stackTrace) { - processFileForElement(element, uniqueFiles); + processFileForElement(element, uniqueFiles, failuresByFilename); - // Process inner stack traces recursively if (element.getInnerStackTrace() != null) { for (StackTraceElement innerElement : element.getInnerStackTrace()) { - processFileForElement(innerElement, uniqueFiles); + processFileForElement(innerElement, uniqueFiles, failuresByFilename); } } } @@ -117,40 +119,63 @@ private void processStackTraces(List> stackTraces, Map uniqueFiles) { + private void processFileForElement(StackTraceElement element, Map uniqueFiles, + Map failuresByFilename) { if (element == null) return; String filename = element.getFilename(); - if (!StringUtil.isEmpty(filename) && fprHandle.getSourceFileMap().containsKey(filename) && !uniqueFiles.containsKey(filename)) { - String internalPath = fprHandle.getSourceFileMap().get(filename); - if (internalPath == null) { return; } // Should not happen due to containsKey check, but safe. + if (!StringUtil.isEmpty(filename) && fprHandle.getSourceFileMap().containsKey(filename) + && !uniqueFiles.containsKey(filename) && !failuresByFilename.containsKey(filename)) { + CachedSourceResult result = sourceFileCache.computeIfAbsent(filename, this::loadSourceFile); + if (result.failure() != null) { + failuresByFilename.put(filename, result.failure()); + } else { + uniqueFiles.put(filename, result.sourceFile().toFile(filename)); + } + } + } + + private CachedSourceResult loadSourceFile(String filename) { + try { + String content = fileUtils.readSourceFileContentStrict(fprHandle, filename); + CachedSourceFile sourceFile = new CachedSourceFile( + fileUtils.appendLineNumbers(content, filename, 0), content.split("\\R", -1).length); + return new CachedSourceResult(sourceFile, null); + } catch (ISourceDecoder.SourceDecodeException e) { + return failedSourceFile(filename, e, AuditSkipReason.SOURCE_FILE_DECODE_FAILED); + } catch (IOException e) { + return failedSourceFile(filename, e, AuditSkipReason.SOURCE_FILE_READ_FAILED); + } + } - Path actualSourcePath = fprHandle.getPath("/" + internalPath); + private CachedSourceResult failedSourceFile(String filename, Exception exception, AuditSkipReason reason) { + logger.warn("Could not read source file content for path {}: {}", filename, exception.getMessage()); + return new CachedSourceResult(null, new SourceFileFailure(filename, exception.getMessage(), reason)); + } + private record CachedSourceFile(String content, int endLine) { + private File toFile(String filename) { File file = new File(); file.setName(filename); file.setSegment(false); file.setStartLine(1); + file.setContent(content); + file.setEndLine(endLine); + return file; + } + } - try { - if (Files.exists(actualSourcePath)) { - byte[] encodedBytes = Files.readAllBytes(actualSourcePath); - String content = new String(encodedBytes); - // Keep line markers in prompt file content; downstream gRPC/template rendering is pass-through. - file.setContent(fileUtils.appendLineNumbers(content, filename, 0)); - file.setEndLine(fileUtils.countLines(actualSourcePath)); - } else { - // This warning is now more accurate. - logger.warn("Source file not found at internal path: {}. This may indicate a corrupt FPR.", actualSourcePath); - file.setContent(""); - file.setEndLine(0); - } - } catch (IOException e) { - logger.warn("Error processing file: {}", filename, e); - file.setContent(""); - file.setEndLine(0); - } - uniqueFiles.put(filename, file); + private record CachedSourceResult(CachedSourceFile sourceFile, SourceFileFailure failure) {} + + public record EnrichmentResult(Map files, List failures) { + public boolean hasFailures() { + return !failures.isEmpty(); + } + } + + public record SourceFileFailure(String filename, String message, AuditSkipReason reason) { + public SourceFileFailure { + Objects.requireNonNull(reason, "reason"); } } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java new file mode 100644 index 00000000000..7b231926127 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java @@ -0,0 +1,82 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Domain factory for {@link ISourceDecoder} instances. Single source of truth for + * token/CSV parsing and default candidate order. CLI converters should delegate here. + */ +public final class SourceDecoders { + public static final String DEFAULT_SOURCE_ENCODINGS = "FPR,UTF-8,ISO-8859-1"; + + private SourceDecoders() {} + + /** + * Map a single token to a decoder: {@code FPR} or a charset name. + * + * @throws IllegalArgumentException if token is blank + * @throws java.nio.charset.IllegalCharsetNameException if charset name is illegal + * @throws java.nio.charset.UnsupportedCharsetException if charset is unsupported + */ + public static ISourceDecoder fromToken(String token) { + if (token == null || token.isBlank()) { + throw new IllegalArgumentException("Source encoding must not be blank"); + } + String trimmed = token.trim(); + if (FprSourceDecoder.TOKEN.equalsIgnoreCase(trimmed)) { + return new FprSourceDecoder(); + } + return new CharsetSourceDecoder(trimmed); + } + + /** + * Parse a comma-separated list of encoding tokens into a single decoder + * (composite when more than one candidate). + */ + public static ISourceDecoder fromCsv(String csv) { + String effective = csv == null || csv.isBlank() ? DEFAULT_SOURCE_ENCODINGS : csv; + List decoders = new ArrayList<>(); + for (String part : effective.split(",")) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) { + decoders.add(fromToken(trimmed)); + } + } + if (decoders.isEmpty()) { + return defaults(); + } + return of(decoders); + } + + /** + * Compose an ordered candidate list into one decoder. + */ + public static ISourceDecoder of(List decoders) { + Objects.requireNonNull(decoders, "decoders must not be null"); + if (decoders.isEmpty()) { + return defaults(); + } + if (decoders.size() == 1) { + return decoders.get(0); + } + return new CompositeSourceDecoder(decoders); + } + + public static ISourceDecoder defaults() { + return fromCsv(DEFAULT_SOURCE_ENCODINGS); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncoder.java new file mode 100644 index 00000000000..93668218dd1 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncoder.java @@ -0,0 +1,64 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CodingErrorAction; + +import com.fortify.cli.aviator._common.exception.AviatorSimpleException; + +/** + * Strict source encode/decode helpers ({@link CodingErrorAction#REPORT}). + * Public {@link #encode} is for write-back; package-private decode is shared by decoders. + */ +public final class SourceEncoder { + private SourceEncoder() {} + + public static byte[] encode(String content, Charset charset, String filename) { + try { + return encodeStrict(content, charset); + } catch (CharacterCodingException e) { + throw new SourceEncodeException( + "Source file '" + filename + "' cannot be encoded using " + charset.name(), e); + } + } + + static String decodeStrict(byte[] bytes, Charset charset) throws CharacterCodingException { + return charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString(); + } + + private static byte[] encodeStrict(String content, Charset charset) throws CharacterCodingException { + ByteBuffer buffer = charset.newEncoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .encode(CharBuffer.wrap(content)); + byte[] result = new byte[buffer.remaining()]; + buffer.get(result); + return result; + } + + public static class SourceEncodeException extends AviatorSimpleException { + private static final long serialVersionUID = 1L; + + public SourceEncodeException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java index 874e192e1f2..e3766515ecd 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java @@ -52,6 +52,8 @@ import com.fortify.cli.aviator.audit.model.AuditResponse; import com.fortify.cli.aviator.audit.model.UserPrompt; import com.fortify.cli.aviator.config.IAviatorLogger; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.grpc.token.DeleteTokenRequest; @@ -123,8 +125,11 @@ public AviatorGrpcClient(ManagedChannel channel, long defaultTimeoutSeconds, IAv this(channel, defaultTimeoutSeconds, logger, 30); } - public CompletableFuture> processBatchRequests(Queue requests, String projectName, String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token, FprHandle fprHandle, List customPriorityOrder) { - AviatorStreamProcessor processor = new AviatorStreamProcessor(this, logger, asyncStub, processingExecutor, pingScheduler, pingIntervalSeconds, defaultTimeoutSeconds, fprHandle); + public CompletableFuture> processBatchRequests(Queue requests, String projectName, + String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token, FprHandle fprHandle, + List customPriorityOrder, ISourceDecoder sourceDecoder, FVDLMetadata fvdlMetadata) { + AviatorStreamProcessor processor = new AviatorStreamProcessor(this, logger, asyncStub, processingExecutor, pingScheduler, + pingIntervalSeconds, defaultTimeoutSeconds, fprHandle, sourceDecoder, fvdlMetadata); CompletableFuture> future = processor.processBatchRequests(requests, projectName, FPRBuildId, SSCApplicationName, SSCApplicationVersion, token, customPriorityOrder); future.whenComplete((res, th) -> processor.close()); return future.exceptionally(ex -> { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java index 89d376801f5..959c68a53f2 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java @@ -17,6 +17,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Queue; import java.util.Set; import java.util.UUID; @@ -36,6 +37,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,8 +48,11 @@ import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; import com.fortify.cli.aviator.audit.QuotaBasedFilter; import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.UserPrompt; import com.fortify.cli.aviator.config.IAviatorLogger; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; import com.fortify.cli.aviator.fpr.utils.SourceCodeEnricher; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; @@ -92,8 +97,16 @@ class AviatorStreamProcessor implements AutoCloseable { private volatile Future processingTask; private final Object retryLock = new Object(); private final FprHandle fprHandle; - - public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, AuditorServiceGrpc.AuditorServiceStub asyncStub, ExecutorService processingExecutor, ScheduledExecutorService pingScheduler, long pingIntervalSeconds, long defaultTimeoutSeconds, FprHandle fprHandle) { + private final ISourceDecoder sourceDecoder; + private final FVDLMetadata fvdlMetadata; + private final SourceCodeEnricher sourceCodeEnricher; + private final AtomicInteger skippedRequests = new AtomicInteger(); + private final AtomicInteger failedRequests = new AtomicInteger(); + + public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, AuditorServiceGrpc.AuditorServiceStub asyncStub, + ExecutorService processingExecutor, ScheduledExecutorService pingScheduler, long pingIntervalSeconds, + long defaultTimeoutSeconds, FprHandle fprHandle, ISourceDecoder sourceDecoder, + FVDLMetadata fvdlMetadata) { this.client = client; this.logger = logger; this.asyncStub = asyncStub; @@ -102,6 +115,9 @@ public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, A this.pingIntervalSeconds = pingIntervalSeconds; this.defaultTimeoutSeconds = defaultTimeoutSeconds; this.fprHandle = fprHandle; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); + this.fvdlMetadata = fvdlMetadata; + this.sourceCodeEnricher = new SourceCodeEnricher(fprHandle, this.sourceDecoder, fvdlMetadata); } public CompletableFuture> processBatchRequests(Queue requests, String projectName, String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token, List customPriorityOrder) { @@ -389,7 +405,7 @@ public void onNext(AuditorResponse response) { response.getRequestId(), instanceId, response.getStatus(), metrics.getDuration()); } - outstandingRequests.decrementAndGet(); + decrementOutstanding(completedWrapper); requestSemaphore.release(); AuditResponse auditResponse = GrpcUtil.convertToAuditResponse(response); @@ -436,11 +452,14 @@ public void onError(Throwable t) { if (infinite || currentStreamState.streamRetryCount < Constants.MAX_STREAM_RETRIES) { LOG.debug("Stream encountered retryable error: {}. Will retry...", t.getMessage()); - int reAdded = inflightRequests.size(); - inflightRequests.values().forEach(processingQueue::addFirst); + int requeuedRequestCount = inflightRequests.size(); + inflightRequests.values().forEach(wrapper -> { + wrapper.outstandingTracked = false; + processingQueue.addFirst(wrapper); + }); inflightRequests.clear(); - outstandingRequests.addAndGet(-reAdded); - requestSemaphore.release(reAdded); + outstandingRequests.addAndGet(-requeuedRequestCount); + requestSemaphore.release(requeuedRequestCount); if (outstandingRequests.get() < 0) { outstandingRequests.set(0); } @@ -707,14 +726,12 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque String instanceId = wrapper.userPrompt.getIssueData().getInstanceID(); - // Lazy Loading of source code files for individual issue - SourceCodeEnricher sourceCodeEnricher = new SourceCodeEnricher(fprHandle); - - Map enrichedFiles = - sourceCodeEnricher.enrichWithSourceCode(wrapper.userPrompt.getStackTrace()); - List sourceCodeFiles = new ArrayList<>(enrichedFiles.values()); - - wrapper.userPrompt.getFiles().addAll(sourceCodeFiles); + SourceCodeEnricher.EnrichmentResult enrichmentResult = enrichSourceCode(wrapper); + if (enrichmentResult != null && enrichmentResult.hasFailures()) { + completeSkippedRequest(wrapper, enrichmentResult, responses, processedRequests, totalRequests, + resultFuture, streamLatch); + continue; + } logger.info("Size of files {}", wrapper.userPrompt.getFiles().size()); logger.info("Size of programming language {}", wrapper.userPrompt.getProgrammingLanguages().size()); @@ -724,8 +741,8 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque continue; } - if (wrapper.attemptCount == 0) { - outstandingRequests.incrementAndGet(); + if (!wrapper.outstandingTracked) { + incrementOutstanding(wrapper); } if (wrapper.attemptCount > 0) { @@ -761,10 +778,11 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque currentStreamState.pendingIssueIds.remove(instanceId); int completed = processedRequests.incrementAndGet(); - outstandingRequests.decrementAndGet(); + int failed = failedRequests.incrementAndGet(); + decrementOutstanding(wrapper); requestSemaphore.release(); - logger.progress("Processed " + completed + " out of " + totalRequests + " issues (1 failed)."); + logger.progress("Processed " + completed + " out of " + totalRequests + " issues (" + failed + " failed)."); } else { LOG.error("Caught AviatorSimpleException but the request wrapper was null.", e); } @@ -792,6 +810,96 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque processingQueue.size(), processedRequests.get(), totalRequests, outstandingRequests.get()); } + private SourceCodeEnricher.EnrichmentResult enrichSourceCode(RequestWrapper wrapper) { + synchronized (wrapper) { + if (wrapper.sourceCodeEnriched) { + return null; + } + + SourceCodeEnricher.EnrichmentResult enrichmentResult = + sourceCodeEnricher.enrichWithSourceCodeDetailed(wrapper.userPrompt.getStackTrace()); + if (!enrichmentResult.hasFailures()) { + List sourceCodeFiles = + new ArrayList<>(enrichmentResult.files().values()); + wrapper.userPrompt.getFiles().addAll(sourceCodeFiles); + wrapper.sourceCodeEnriched = true; + } + return enrichmentResult; + } + } + + private void incrementOutstanding(RequestWrapper wrapper) { + if (!wrapper.outstandingTracked) { + outstandingRequests.incrementAndGet(); + wrapper.outstandingTracked = true; + } + } + + private void decrementOutstanding(RequestWrapper wrapper) { + if (wrapper != null && wrapper.outstandingTracked) { + wrapper.outstandingTracked = false; + if (outstandingRequests.decrementAndGet() < 0) { + outstandingRequests.set(0); + } + } + } + + private void completeSkippedRequest(RequestWrapper wrapper, SourceCodeEnricher.EnrichmentResult enrichmentResult, + Map responses, AtomicInteger processedRequests, + int totalRequests, CompletableFuture> resultFuture, + CountDownLatch streamLatch) { + String instanceId = wrapper.userPrompt.getIssueData().getInstanceID(); + AuditSkipReason skipReason = enrichmentResult.failures().get(0).reason(); + String failureMessage = formatSourceFailureMessage(enrichmentResult, skipReason); + AuditResponse skippedResponse = new AuditResponse(); + skippedResponse.setIssueId(instanceId); + skippedResponse.setStatus("SKIPPED"); + skippedResponse.setAuditSkipReason(skipReason); + skippedResponse.setStatusMessage(failureMessage); + responses.put(instanceId, skippedResponse); + + currentStreamState.processedIssueIds.add(instanceId); + currentStreamState.pendingIssueIds.remove(instanceId); + decrementOutstanding(wrapper); + requestSemaphore.release(); + + int completed = processedRequests.incrementAndGet(); + int skipped = skippedRequests.incrementAndGet(); + LOG.warn("Skipping issue {}: {}", instanceId, failureMessage); + logger.progress("Processed " + completed + " out of " + totalRequests + " issues (" + skipped + " skipped)."); + + if (completed >= totalRequests) { + logger.info("All requests accounted for, completing stream."); + if (requestHandler != null && !requestHandler.isCompleted()) { + requestHandler.complete(); + } + streamLatch.countDown(); + if (!resultFuture.isDone()) { + resultFuture.complete(responses); + } + } + } + + private String formatSourceFailureMessage(SourceCodeEnricher.EnrichmentResult enrichmentResult, + AuditSkipReason skipReason) { + List filenames = enrichmentResult.failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::filename) + .distinct() + .collect(Collectors.toList()); + String details = enrichmentResult.failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::message) + .filter(message -> message != null && !message.isBlank()) + .distinct() + .collect(Collectors.joining("; ")); + String detailSuffix = details.isBlank() ? "" : " (" + details + ")"; + return switch (skipReason) { + case SOURCE_FILE_DECODE_FAILED -> skipReason.format( + filenames.size() == 1 ? "" : "s", String.join(", ", filenames), detailSuffix); + case SOURCE_FILE_READ_FAILED -> skipReason.format(String.join(", ", filenames), detailSuffix); + case SKIPPED_BY_AVIATOR -> details.isBlank() ? skipReason.getDisplayMessage() : details; + }; + } + private void handleServerBusy(String requestId, int totalRequests, AtomicInteger processedRequests, Map responses, CompletableFuture> resultFuture, CountDownLatch streamLatch) { RequestWrapper wrapperToRetry = inflightRequests.remove(requestId); if (wrapperToRetry == null) { @@ -811,11 +919,13 @@ private void handleServerBusy(String requestId, int totalRequests, AtomicInteger failedResponse.setIssueId(wrapperToRetry.userPrompt.getIssueData().getInstanceID()); failedResponse.setStatus("RETRY_LIMIT_EXCEEDED"); failedResponse.setStatusMessage("Request failed after " + Constants.MAX_RETRIES + " retries due to server overload."); + failedResponse.setSubmittedToAviator(true); responses.put(wrapperToRetry.userPrompt.getIssueData().getInstanceID(), failedResponse); int completed = processedRequests.incrementAndGet(); logger.progress("Request permanently failed due to server busy - Processed " + completed + " out of " + totalRequests + " issues"); - int stillOutstanding = outstandingRequests.decrementAndGet(); + decrementOutstanding(wrapperToRetry); + int stillOutstanding = outstandingRequests.get(); LOG.warn("WARN: Request for instance {} permanently failed. Remaining outstanding requests: {}", wrapperToRetry.userPrompt.getIssueData().getInstanceID(), stillOutstanding); if (completed >= totalRequests) { logger.info("All requests accounted for after permanent failure, completing stream."); @@ -848,7 +958,7 @@ private void submitUserPrompt(RequestWrapper wrapper) { LOG.error("Failed to send request for instance {} after all retries. Re-queueing for later attempt.", wrapper.userPrompt.getIssueData().getInstanceID()); inflightRequests.remove(requestId); requestMetricsMap.remove(requestId); - outstandingRequests.decrementAndGet(); + decrementOutstanding(wrapper); requestSemaphore.release(); wrapper.attemptCount++; processingQueue.addLast(wrapper); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java index 2d44b56f599..1fc5785ca33 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java @@ -29,6 +29,7 @@ import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.Autoremediation; import com.fortify.cli.aviator.audit.model.Change; import com.fortify.cli.aviator.audit.model.StackTraceElement; @@ -195,7 +196,11 @@ static AuditResponse convertToAuditResponse(AuditorResponse response) { auditResponse.setOutputToken(response.getOutputToken()); auditResponse.setStatus(response.getStatus()); auditResponse.setStatusMessage(response.getStatusMessage()); + if ("SKIPPED".equalsIgnoreCase(response.getStatus())) { + auditResponse.setAuditSkipReason(AuditSkipReason.SKIPPED_BY_AVIATOR); + } auditResponse.setIssueId(response.getIssueId()); + auditResponse.setSubmittedToAviator(true); auditResponse.setTier(response.getTier()); auditResponse.setAviatorPredictionTag(response.getAviatorPredictionTag()); auditResponse.setIsAviatorProcessed(response.getIsAviatorProcessed()); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java index 954f048f4b2..65ddee600c5 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java @@ -17,6 +17,8 @@ class RequestWrapper { final UserPrompt userPrompt; int attemptCount = 0; + volatile boolean outstandingTracked; + volatile boolean sourceCodeEnriched; RequestWrapper(UserPrompt userPrompt) { this.userPrompt = userPrompt; diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverterTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverterTest.java new file mode 100644 index 00000000000..0ad64fab89d --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverterTest.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator._common.cli.converter; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +import picocli.CommandLine.TypeConversionException; + +/** + * Only the CLI adapter layer: domain factory exceptions become {@link TypeConversionException}. + * Happy-path token semantics are covered by {@code SourceDecodersTest}. + */ +class SourceDecoderConverterTest { + + private final SourceDecoderConverter converter = new SourceDecoderConverter(); + + @Test + void convert_delegatesHappyPathToFactory() { + assertEquals("FPR", converter.convert("FPR").describe()); + assertEquals("UTF-8", converter.convert("UTF-8").describe()); + } + + @Test + void convert_mapsFactoryFailuresToTypeConversionException() { + assertThrows(TypeConversionException.class, () -> converter.convert(" ")); + assertThrows(TypeConversionException.class, () -> converter.convert("NOT-A-REAL-CHARSET")); + } +} diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java new file mode 100644 index 00000000000..0dc06bc834f --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java @@ -0,0 +1,86 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.audit; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; + +class AuditFprStatusTest { + + @Test + void determinesAuditStatusForSuccessSkipAndFailureCombinations() { + assertEquals("AUDITED", AuditFPR.determineAuditStatus(3, 0, 3, 3)); + assertEquals("PARTIALLY_AUDITED", AuditFPR.determineAuditStatus(2, 1, 3, 3)); + assertEquals("SKIPPED", AuditFPR.determineAuditStatus(0, 3, 3, 3)); + assertEquals("FAILED", AuditFPR.determineAuditStatus(0, 2, 3, 3)); + assertEquals("FAILED", AuditFPR.determineAuditStatus(0, 3, 3, 2)); + } + + @Test + void countsOnlyExplicitSkippedResponsesAsSkipped() { + AuditResponse skipped = AuditResponse.builder() + .status("SKIPPED") + .auditSkipReason(AuditSkipReason.SOURCE_FILE_DECODE_FAILED) + .statusMessage("The source decoder wording can change") + .build(); + AuditResponse failed = AuditResponse.builder() + .status("FAILED") + .statusMessage("backend error") + .build(); + AuditResponse success = AuditResponse.builder().status("SUCCESS").build(); + + assertEquals(Map.of("Source file decode failed", 1), + AuditFPR.getSkippedAuditReasons(Map.of( + "skipped", skipped, + "failed", failed, + "success", success), 3)); + } + + @Test + void preservesMissingResponseAccountingForFilteredIssues() { + AuditResponse success = AuditResponse.builder().status("SUCCESS").build(); + + assertEquals(Map.of("No audit response received", 1), + AuditFPR.getSkippedAuditReasons(Map.of("success", success), 2)); + } + + @Test + void countsOnlyResponsesOriginatingFromAviatorAsSubmitted() { + AuditResponse localSkip = AuditResponse.builder().status("SKIPPED").build(); + AuditResponse localFailure = AuditResponse.builder() + .status("FAILED") + .statusMessage("Request validation failed") + .build(); + AuditResponse serverSkip = AuditResponse.builder() + .status("SKIPPED") + .submittedToAviator(true) + .build(); + AuditResponse serverFailure = AuditResponse.builder() + .status("FAILED") + .statusMessage("Aviator processing failed") + .submittedToAviator(true) + .build(); + + assertEquals(2, AuditFPR.getSubmittedAuditCount(Map.of( + "local", localSkip, + "localFailure", localFailure, + "server", serverSkip, + "serverFailure", serverFailure))); + } +} \ No newline at end of file diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java new file mode 100644 index 00000000000..3e42be508d8 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java @@ -0,0 +1,267 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; +import com.fortify.cli.aviator.audit.model.StackTraceElement; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.util.FprHandle; + +class SourceCodeEnricherTest { + + @TempDir + Path tempDir; + + @Test + void reportsEveryUniqueDecodeFailureAndDoesNotHideSuccessfulFiles() throws Exception { + Map sourceFiles = new LinkedHashMap<>(); + sourceFiles.put("good.java", "class Good {}".getBytes(StandardCharsets.UTF_8)); + sourceFiles.put("bad-one.java", new byte[] {(byte) 0xFF}); + sourceFiles.put("bad-two.java", new byte[] {(byte) 0xFE}); + Path fprPath = createFpr(sourceFiles); + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + SourceCodeEnricher.EnrichmentResult result = new SourceCodeEnricher( + fprHandle, SourceDecoders.fromToken("UTF-8"), null) + .enrichWithSourceCodeDetailed(List.of(List.of( + element("good.java"), + element("bad-one.java"), + element("bad-two.java"), + element("bad-one.java")))); + + assertTrue(result.hasFailures()); + assertEquals(List.of("bad-one.java", "bad-two.java"), + result.failures().stream().map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(List.of(AuditSkipReason.SOURCE_FILE_DECODE_FAILED, AuditSkipReason.SOURCE_FILE_DECODE_FAILED), + result.failures().stream().map(SourceCodeEnricher.SourceFileFailure::reason).toList()); + assertEquals(List.of("good.java"), result.files().keySet().stream().toList()); + } + } + + @Test + void classifiesReadFailuresSeparatelyFromDecodeFailures() throws Exception { + Path fprPath = createFprWithMissingSource("missing.java"); + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + SourceCodeEnricher.EnrichmentResult result = new SourceCodeEnricher( + fprHandle, SourceDecoders.fromToken("UTF-8"), null) + .enrichWithSourceCodeDetailed(List.of(List.of(element("missing.java")))); + + assertEquals(AuditSkipReason.SOURCE_FILE_READ_FAILED, result.failures().get(0).reason()); + } + } + + @Test + void cachesSuccessfulAndFailedLoadsAcrossEnrichmentCalls() throws Exception { + Map sourceFiles = new LinkedHashMap<>(); + sourceFiles.put("good.java", "class Good {}".getBytes(StandardCharsets.UTF_8)); + sourceFiles.put("bad.java", new byte[] {(byte) 0xFF}); + Path fprPath = createFpr(sourceFiles); + AtomicInteger decodeCalls = new AtomicInteger(); + ISourceDecoder decoder = new ISourceDecoder() { + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata metadata) { + decodeCalls.incrementAndGet(); + if (bytes.length > 0 && bytes[0] == (byte) 0xFF) { + throw new SourceDecodeException("decode failed"); + } + return new DecodeResult(new String(bytes, StandardCharsets.UTF_8), StandardCharsets.UTF_8, "test"); + } + + @Override + public String describe() { + return "test"; + } + }; + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + SourceCodeEnricher enricher = new SourceCodeEnricher(fprHandle, decoder, null); + List> stackTraces = List.of(List.of(element("good.java"), element("bad.java"))); + + SourceCodeEnricher.EnrichmentResult firstResult = enricher.enrichWithSourceCodeDetailed(stackTraces); + SourceCodeEnricher.EnrichmentResult secondResult = enricher.enrichWithSourceCodeDetailed(stackTraces); + + assertEquals(2, decodeCalls.get()); + assertEquals(List.of("good.java"), secondResult.files().keySet().stream().toList()); + assertEquals(List.of("bad.java"), secondResult.failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(firstResult.files().get("good.java").getContent(), secondResult.files().get("good.java").getContent()); + } + } + + @Test + void loadsAFileOnceWhenEnrichmentCallsAreConcurrent() throws Exception { + Map sourceFiles = Map.of("good.java", "class Good {}".getBytes(StandardCharsets.UTF_8)); + Path fprPath = createFpr(sourceFiles); + AtomicInteger decodeCalls = new AtomicInteger(); + CountDownLatch decodeStarted = new CountDownLatch(1); + CountDownLatch allowDecode = new CountDownLatch(1); + ISourceDecoder decoder = new ISourceDecoder() { + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata metadata) { + decodeCalls.incrementAndGet(); + decodeStarted.countDown(); + try { + if (!allowDecode.await(5, TimeUnit.SECONDS)) { + throw new SourceDecodeException("Timed out waiting to decode"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SourceDecodeException("Interrupted while decoding", e); + } + return new DecodeResult(new String(bytes, StandardCharsets.UTF_8), StandardCharsets.UTF_8, "test"); + } + + @Override + public String describe() { + return "test"; + } + }; + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + SourceCodeEnricher enricher = new SourceCodeEnricher(fprHandle, decoder, null); + List> stackTraces = List.of(List.of(element("good.java"))); + Future first = executor.submit( + () -> enricher.enrichWithSourceCodeDetailed(stackTraces)); + Future second = executor.submit( + () -> enricher.enrichWithSourceCodeDetailed(stackTraces)); + + assertTrue(decodeStarted.await(5, TimeUnit.SECONDS)); + allowDecode.countDown(); + + assertEquals(List.of("good.java"), first.get(5, TimeUnit.SECONDS).files().keySet().stream().toList()); + assertEquals(List.of("good.java"), second.get(5, TimeUnit.SECONDS).files().keySet().stream().toList()); + assertEquals(1, decodeCalls.get()); + } finally { + executor.shutdownNow(); + } + } + } + + @Test + void cachesConcurrentDecodeFailureOnce() throws Exception { + Map sourceFiles = Map.of("bad.java", new byte[] {(byte) 0xFF}); + Path fprPath = createFpr(sourceFiles); + AtomicInteger decodeCalls = new AtomicInteger(); + CountDownLatch decodeStarted = new CountDownLatch(1); + CountDownLatch allowDecode = new CountDownLatch(1); + ISourceDecoder decoder = new ISourceDecoder() { + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata metadata) { + decodeCalls.incrementAndGet(); + decodeStarted.countDown(); + try { + if (!allowDecode.await(5, TimeUnit.SECONDS)) { + throw new SourceDecodeException("Timed out waiting to decode"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SourceDecodeException("Interrupted while decoding", e); + } + throw new SourceDecodeException("decode failed"); + } + + @Override + public String describe() { + return "test"; + } + }; + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + SourceCodeEnricher enricher = new SourceCodeEnricher(fprHandle, decoder, null); + List> stackTraces = List.of(List.of(element("bad.java"))); + Future first = executor.submit( + () -> enricher.enrichWithSourceCodeDetailed(stackTraces)); + Future second = executor.submit( + () -> enricher.enrichWithSourceCodeDetailed(stackTraces)); + + assertTrue(decodeStarted.await(5, TimeUnit.SECONDS)); + allowDecode.countDown(); + + assertEquals(List.of("bad.java"), first.get(5, TimeUnit.SECONDS).failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(List.of("bad.java"), second.get(5, TimeUnit.SECONDS).failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(1, decodeCalls.get()); + } finally { + executor.shutdownNow(); + } + } + } + + private static StackTraceElement element(String filename) { + return new StackTraceElement(filename, 1, "", "", null, null, null); + } + + private Path createFpr(Map sourceFiles) throws IOException { + Path fprPath = tempDir.resolve("source-enricher-test.fpr"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(fprPath))) { + writeEntry(zip, "src-archive/index.xml", createIndex(sourceFiles.keySet()).getBytes(StandardCharsets.UTF_8)); + int index = 1; + for (Map.Entry sourceFile : sourceFiles.entrySet()) { + writeEntry(zip, "src-archive/" + index++, sourceFile.getValue()); + } + } + return fprPath; + } + + private Path createFprWithMissingSource(String filename) throws IOException { + Path fprPath = tempDir.resolve("source-enricher-missing-source.fpr"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(fprPath))) { + writeEntry(zip, "src-archive/index.xml", createIndex(List.of(filename)).getBytes(StandardCharsets.UTF_8)); + } + return fprPath; + } + + private static String createIndex(Iterable filenames) { + StringBuilder index = new StringBuilder(""); + int entry = 1; + for (String filename : filenames) { + index.append("src-archive/") + .append(entry++).append(""); + } + return index.append("").toString(); + } + + private static void writeEntry(ZipOutputStream zip, String name, byte[] bytes) throws IOException { + zip.putNextEntry(new ZipEntry(name)); + zip.write(bytes); + zip.closeEntry(); + } +} \ No newline at end of file diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java new file mode 100644 index 00000000000..bdc8f254957 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java @@ -0,0 +1,126 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.charset.UnsupportedCharsetException; + +import org.junit.jupiter.api.Test; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.DecodeResult; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.SourceDecodeException; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder.SourceEncodeException; + +/** + * Behavior tests for source encoding factory + decode/encode. + * Prefer real byte decode outcomes over describe()-only checks. + */ +class SourceDecodersTest { + + @Test + void fromToken_rejectsBlankAndUnknownCharset() { + assertThrows(IllegalArgumentException.class, () -> SourceDecoders.fromToken(" ")); + assertThrows(UnsupportedCharsetException.class, () -> SourceDecoders.fromToken("NOT-A-CHARSET")); + } + + @Test + void defaults_matchDocumentedCandidateOrder() { + assertEquals("FPR,UTF-8,ISO-8859-1", SourceDecoders.DEFAULT_SOURCE_ENCODINGS); + assertEquals(SourceDecoders.DEFAULT_SOURCE_ENCODINGS, SourceDecoders.defaults().describe()); + } + + @Test + void decode_fallsThroughFprWhenMetadataMissing() { + byte[] utf8 = "hello".getBytes(StandardCharsets.UTF_8); + DecodeResult result = SourceDecoders.defaults().decode(utf8, "Main.java", null); + assertEquals("hello", result.content()); + assertEquals(StandardCharsets.UTF_8, result.charset()); + assertEquals("UTF-8", result.source()); + } + + @Test + void decode_fallsThroughUtf8ToIso88591ByDefault() { + byte[] iso88591 = {(byte) 0xE9}; + + DecodeResult result = SourceDecoders.defaults().decode(iso88591, "Main.java", null); + + assertEquals("\u00e9", result.content()); + assertEquals(StandardCharsets.ISO_8859_1, result.charset()); + assertEquals("ISO-8859-1", result.source()); + } + + @Test + void decode_usesFprEncodingWhenMetadataPresent() { + FVDLMetadata metadata = new FVDLMetadata(); + metadata.registerSourceFileEncoding("src/Main.java", "ISO-8859-1"); + byte[] latin1 = "caf\u00e9".getBytes(StandardCharsets.ISO_8859_1); + + DecodeResult result = SourceDecoders.defaults().decode(latin1, "src/Main.java", metadata); + + assertEquals("caf\u00e9", result.content()); + assertEquals(StandardCharsets.ISO_8859_1, result.charset()); + assertTrue(result.source().startsWith("FPR("), result.source()); + } + + @Test + void decode_usesWindows1252FromFprMetadata() { + FVDLMetadata metadata = new FVDLMetadata(); + metadata.registerSourceFileEncoding("payments.c", "windows-1252"); + byte[] windows1252 = new byte[] {(byte) 0x93, 'p', 'r', 'e', 'm', 'i', 'u', 'm', (byte) 0x94}; + + DecodeResult result = SourceDecoders.defaults().decode(windows1252, "payments.c", metadata); + + assertEquals("\u201cpremium\u201d", result.content()); + assertEquals("windows-1252", result.charset().name()); + assertEquals("FPR(windows-1252)", result.source()); + } + + @Test + void decode_allCandidatesFail_messageListsAttempts() { + ISourceDecoder decoder = SourceDecoders.fromCsv("UTF-8,US-ASCII"); + byte[] invalid = new byte[] {(byte) 0xFF, (byte) 0xFE, (byte) 0x00}; + + SourceDecodeException ex = assertThrows(SourceDecodeException.class, + () -> decoder.decode(invalid, "bad.bin", null)); + + assertTrue(ex.getMessage().contains("bad.bin"), ex.getMessage()); + assertTrue(ex.getMessage().contains("UTF-8"), ex.getMessage()); + assertTrue(ex.getMessage().contains("US-ASCII"), ex.getMessage()); + } + + @Test + void encode_roundTripAndRejectsUnmappable() { + String content = "secure code"; + byte[] encoded = SourceEncoder.encode(content, StandardCharsets.UTF_8, "a.java"); + assertEquals(content, SourceDecoders.fromToken("UTF-8").decode(encoded, "a.java", null).content()); + + assertThrows(SourceEncodeException.class, + () -> SourceEncoder.encode("caf\u00e9", StandardCharsets.US_ASCII, "a.java")); + } + + @Test + void fromToken_fprIsCaseInsensitive() { + assertEquals("FPR", SourceDecoders.fromToken("fpr").describe()); + } + + @Test + void fromCsv_compositesMultipleCandidates() { + assertEquals("UTF-8,CP850", SourceDecoders.fromCsv("UTF-8,CP850").describe()); + assertEquals("UTF-8", SourceDecoders.fromCsv("UTF-8").describe()); + } +} diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/grpc/GrpcUtilTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/grpc/GrpcUtilTest.java index 2f503d3d0a5..09285bae9e9 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/grpc/GrpcUtilTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/grpc/GrpcUtilTest.java @@ -19,6 +19,9 @@ import org.junit.jupiter.api.Test; +import com.fortify.aviator.grpc.AuditorResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.File; import com.fortify.cli.aviator.audit.model.UserPrompt; @@ -43,4 +46,16 @@ void shouldPreserveResolvedProgrammingLanguages() { assertEquals(List.of("PLSQL"), auditRequest.getProgrammingLanguagesList()); assertEquals("PLSQL", auditRequest.getLanguage()); } + + @Test + void shouldClassifySkippedServerResponseWithoutInspectingMessage() { + AuditorResponse response = AuditorResponse.newBuilder() + .setStatus("SKIPPED") + .setStatusMessage("example could not be read from the FPR") + .build(); + + AuditResponse auditResponse = GrpcUtil.convertToAuditResponse(response); + + assertEquals(AuditSkipReason.SKIPPED_BY_AVIATOR, auditResponse.getAuditSkipReason()); + } } \ No newline at end of file diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java index 30fe3af0ae2..c6bff9c596e 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java @@ -26,6 +26,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.JsonNode; +import com.fortify.cli.aviator._common.cli.mixin.SourceEncodingsMixin; import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator.applyRemediation.ApplyAutoRemediationOnSource; import com.fortify.cli.aviator.config.AviatorLoggerImpl; @@ -63,6 +64,7 @@ public class AviatorSSCApplyRemediationsCommand extends AbstractSSCJsonNodeOutpu private static final Logger LOG = LoggerFactory.getLogger(AviatorSSCApplyRemediationsCommand.class); @Option(names = {"--source-dir"}, descriptionKey = "fcli.aviator.ssc.apply-remediations.source-dir") private String sourceCodeDirectory = System.getProperty("user.dir"); + @Mixin private SourceEncodingsMixin sourceEncodingsMixin; @Override @SneakyThrows @@ -128,7 +130,8 @@ JsonNode processAllAviatorArtifacts(OffsetDateTime sinceDate) { try { fprPath = downloadArtifactFpr(ad); try (FprHandle fprHandle = new FprHandle(fprPath)) { - var metric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, logger); + var metric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, + sourceEncodingsMixin.getSourceDecoder(), logger); totalRemediations += metric.totalRemediations(); appliedRemediations += metric.appliedRemediations(); skippedRemediations += metric.skippedRemediations(); @@ -181,7 +184,8 @@ JsonNode processFprRemediations(SSCArtifactDescriptor ad) { try { logger.progress("Status: Processing FPR with Aviator for Applying Auto Remediations"); try (FprHandle fprHandle = new FprHandle(fprPath)) { - var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, logger); + var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, + sourceEncodingsMixin.getSourceDecoder(), logger); String status = remediationMetric.appliedRemediations() > 0 ? "Remediation-Applied" : "No-Remediation-Applied"; diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java index 2a4e73281df..5a0c876ef9e 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.aviator._common.cli.mixin.SourceEncodingsMixin; import com.fortify.cli.aviator._common.config.AviatorConfigManager; import com.fortify.cli.aviator._common.session.user.cli.mixin.AviatorUserSessionDescriptorSupplier; import com.fortify.cli.aviator._common.session.user.helper.AviatorUserSessionDescriptor; @@ -79,6 +80,7 @@ public class AviatorSSCAuditCommand extends AbstractSSCJsonNodeOutputCommand imp @ArgGroup(exclusive = true, multiplicity = "0..1") private QuotaHandlingArgGroup quotaHandlingArgGroup = new QuotaHandlingArgGroup(); @Option(names = {"--test-exceeding-quota"}) private boolean testExceedingQuota; @Option(names = {"--default-quota-fallback"}) private boolean defaultQuotaFallback; + @Mixin private SourceEncodingsMixin sourceEncodingsMixin; private static final Logger LOG = LoggerFactory.getLogger(AviatorSSCAuditCommand.class); private Long checkedQuotaBefore; @@ -275,6 +277,7 @@ private JsonNode processFpr(UnirestInstance unirest, SSCAppVersionDescriptor av, .noFilterSet(isNoFilterSet()) .folderNames(folderNames) .folderPriorityOrder(getFolderPriorityOrder()) + .sourceDecoder(sourceEncodingsMixin.getSourceDecoder()) .build()); } catch (Exception e) { LOG.error("FPR audit failed for {}:{}: {}", av.getApplicationName(), av.getVersionName(), e.getMessage(), e); diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java index 542ee85bb86..e9ce3228879 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java @@ -135,12 +135,34 @@ public static void setAuditStats(ObjectNode result, FPRAuditResult auditResult) break; } audit.put("message", message); - audit.put("submitted", auditResult.getTotalIssuesToAudit()); + audit.put("submitted", auditResult.getIssuesSubmitted()); audit.put("succeeded", auditResult.getIssuesSuccessfullyAudited()); - audit.put("skipped", Math.max(0, auditResult.getTotalIssuesToAudit() - auditResult.getIssuesSuccessfullyAudited())); + audit.put("skipped", auditResult.getIssuesSkipped()); + audit.put("skippedReasons", formatSkippedReasons(auditResult.getSkippedByReason())); + audit.set("skippedByReason", toObjectNode(auditResult.getSkippedByReason())); + audit.put("remediationGenerationSkipped", auditResult.getRemediationGenerationSkipped()); + audit.put("remediationGenerationSkippedReasons", formatSkippedReasons(auditResult.getRemediationGenerationSkippedByReason())); + audit.set("remediationGenerationSkippedByReason", toObjectNode(auditResult.getRemediationGenerationSkippedByReason())); ((ObjectNode) result.get("operation")).set("audit", audit); } + private static ObjectNode toObjectNode(Map skippedByReason) { + ObjectNode object = JsonHelper.getObjectMapper().createObjectNode(); + if (skippedByReason != null) { + skippedByReason.forEach(object::put); + } + return object; + } + + private static String formatSkippedReasons(Map skippedByReason) { + if (skippedByReason == null || skippedByReason.isEmpty()) { + return ""; + } + List parts = new ArrayList<>(); + skippedByReason.forEach((reason, count) -> parts.add(reason + "=" + count)); + return String.join(", ", parts); + } + /** * Sets only the {@code operation.audit.message} field without audit stats. * Used for code paths that don't perform an actual audit (SKIPPED, FAILED, QUOTA_EXCEEDED, etc.). @@ -175,7 +197,8 @@ public static void setAvailableQuotaBefore(ObjectNode result, long quota) { public static String getProgressMessage(FPRAuditResult auditResult) { switch (auditResult.getStatus()) { case "SKIPPED": - return "No issues to audit, skipping upload"; + return (auditResult.getMessage() != null ? auditResult.getMessage() : "No issues to audit") + + ", skipping upload"; case "FAILED": String message = auditResult.getMessage() != null ? auditResult.getMessage() : "Unknown error"; return "Audit failed: " + message; diff --git a/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties b/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties index da3da97cfb2..7a2c64f68e3 100644 --- a/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties +++ b/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties @@ -129,6 +129,7 @@ fcli.aviator.ssc.audit.skip-if-exceeding-quota = Skip audit if the number of ope fcli.aviator.ssc.audit.test-exceeding-quota = Check whether the number of open issues exceeds the available Fortify Remediation Aviator quota and report the result without performing an audit. fcli.aviator.ssc.audit.default-quota-fallback = (Internal) When the Fortify Aviator application does not exist, use the tenant default quota instead of reporting app not found. Used by bulk audit. fcli.aviator.ssc.audit.folder-priority-order = Custom priority order for folder-based filtering when quota is exceeded (comma-separated, highest priority first). Example: Critical,High,Medium,Low. If not specified, uses default priority order. +fcli.aviator.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files. Use FPR to try the source encoding recorded in audit.fvdl. When writing remediated files, the accepted encoding is used. Default value: ${DEFAULT-VALUE}. fcli.aviator.ssc.audit.refresh = By default, this command will refresh the source application version's metrics when copying from it. \ Note that for large applications this can lead to an error if the timeout expires. fcli.aviator.ssc.audit.refresh-timeout = Time-out, for example 30s (30 seconds), 5m (5 minutes), 1h (1 hour). Default value: ${DEFAULT-VALUE} @@ -204,7 +205,7 @@ fcli.env.default.prefix=FCLI_DEFAULT # Table output columns configuration fcli.aviator.session.output.table.args = name,type,url,created,expires,expired fcli.aviator.admin-config.output.table.args = name,type,url,created -fcli.aviator.ssc.audit.output.table.args = id,application.name,name,artifactId,action +fcli.aviator.ssc.audit.output.table.args = id,application.name,name,artifactId,action,operation.audit.skippedReasons fcli.aviator.app.create.output.table.args = id,name,entitlement_id,disclaimer,quota_last_updated,quota fcli.aviator.app.add-entitlement.output.table.args = id,name,entitlement_id,entitlement_multiplier,entitlements_consumed,quota_last_updated,quota fcli.aviator.app.delete.output.table.args = message diff --git a/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java b/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java index e98fa9a5587..d8988e8a860 100644 --- a/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java +++ b/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java @@ -19,9 +19,15 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Map; import org.junit.jupiter.api.Test; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.aviator.audit.model.FPRAuditResult; +import com.fortify.cli.aviator.ssc.helper.AviatorSSCAuditHelper; +import com.fortify.cli.common.json.JsonHelper; + import picocli.CommandLine; class AviatorSSCAuditCommandTest { @@ -46,6 +52,48 @@ void testAllowsNoFilterSetOption() { assertNull(cmd.getFilterSetTitleOrId()); } + @Test + void reportsDecodeSkippedIssuesAsNotSubmitted() { + ObjectNode result = JsonHelper.getObjectMapper().createObjectNode(); + result.set("operation", JsonHelper.getObjectMapper().createObjectNode()); + var auditResult = new FPRAuditResult(null, "SKIPPED", "All 6 issues were skipped", 0, 6, 0, 6, + Map.of("Source file decode failed", 6), 0, Map.of()); + + AviatorSSCAuditHelper.setAuditStats(result, auditResult); + + var audit = result.path("operation").path("audit"); + assertEquals(0, audit.get("submitted").asInt()); + assertEquals(0, audit.get("succeeded").asInt()); + assertEquals(6, audit.get("skipped").asInt()); + } + + @Test + void reportsServerSkippedIssuesAsSubmitted() { + ObjectNode result = JsonHelper.getObjectMapper().createObjectNode(); + result.set("operation", JsonHelper.getObjectMapper().createObjectNode()); + var auditResult = new FPRAuditResult(null, "SKIPPED", "Skipped by Aviator", 0, 1, 1, 1, + Map.of("Skipped by Aviator", 1), 0, Map.of()); + + AviatorSSCAuditHelper.setAuditStats(result, auditResult); + + assertEquals(1, result.path("operation").path("audit").get("submitted").asInt()); + } + + @Test + void excludesOnlyDecodeSkippedIssuesFromSubmittedCount() { + ObjectNode result = JsonHelper.getObjectMapper().createObjectNode(); + result.set("operation", JsonHelper.getObjectMapper().createObjectNode()); + var auditResult = new FPRAuditResult(null, "PARTIALLY_AUDITED", null, 2, 4, 3, 2, + Map.of("Source file decode failed", 1, "Skipped by Aviator", 1), 0, Map.of()); + + AviatorSSCAuditHelper.setAuditStats(result, auditResult); + + var audit = result.path("operation").path("audit"); + assertEquals(3, audit.get("submitted").asInt()); + assertEquals(2, audit.get("succeeded").asInt()); + assertEquals(2, audit.get("skipped").asInt()); + } + private static AviatorSSCAuditCommand parse(String... args) { var cmd = new AviatorSSCAuditCommand(); var fullArgs = new ArrayList(); diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java index f792edbef1e..471232a1e03 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java @@ -20,6 +20,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.JsonNode; +import com.fortify.cli.aviator._common.cli.mixin.SourceEncodingsMixin; import com.fortify.cli.aviator.applyRemediation.ApplyAutoRemediationOnSource; import com.fortify.cli.aviator.config.AviatorLoggerImpl; import com.fortify.cli.aviator.util.FprHandle; @@ -56,6 +57,7 @@ public class FoDAviatorApplyRemediationsCommand extends AbstractFoDJsonNodeOutpu @Mixin private FoDReleaseByQualifiedNameOrIdResolverMixin.RequiredOption releaseResolver; private static final Logger LOG = LoggerFactory.getLogger(FoDAviatorApplyRemediationsCommand.class); @Option(names = {"--source-dir"}) private String sourceCodeDirectory = System.getProperty("user.dir"); + @Mixin private SourceEncodingsMixin sourceEncodingsMixin; @Override @SneakyThrows public JsonNode getJsonNode(UnirestInstance unirest) { @@ -82,7 +84,8 @@ private JsonNode processFprRemediations(UnirestInstance unirest, FoDReleaseDescr logger.progress("Status: Processing FPR with Aviator for Applying Auto Remediations"); try (FprHandle fprHandle = new FprHandle(downloadedFprPath)) { - var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, logger); + var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, + sourceEncodingsMixin.getSourceDecoder(), logger); LOG.info("Applied remediation {}", remediationMetric.appliedRemediations()); LOG.info("Total remediation {}", remediationMetric.totalRemediations()); String status = remediationMetric.appliedRemediations() > 0 ? "Remediation-Applied" : "No-Remediation-Applied"; diff --git a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties index 15976845f03..6248bee37db 100644 --- a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties +++ b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties @@ -1040,6 +1040,7 @@ fcli.fod.attribute.update.values = List of picklist values (only for Picklist da fcli.fod.aviator.usage.header = Use Fortify Remediation Aviator with FoD. fcli.fod.aviator.apply-remediations.usage.header = Apply Fortify Remediation Aviator auto-remediations to source code. fcli.fod.aviator.apply-remediations.source-dir = Directory containing source code to apply remediations to. Default value: ${DEFAULT-VALUE}. +fcli.aviator.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files. Use FPR to try the source encoding recorded in audit.fvdl. When writing remediated files, the accepted encoding is used. Default value: ${DEFAULT-VALUE}. # various messages displayed during execution From fbf4f7bcbd20f24e56445a2b70adc056d7e115fc Mon Sep 17 00:00:00 2001 From: Ruud Senden <8635138+rsenden@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:56:20 +0200 Subject: [PATCH 03/37] chore: Restore FoD regressions from Aviator 26.2 merge (#1070) fix: `fcli fod dast-scan start`: Restore fix to allow DAST scan to start if no prior scans exist (lost in Aviator 26.2 merge) (fixes #1068) fix: `fcli fod dast-scan start`: Restore `--vpn` support and 422 active-scan fallback (lost in Aviator 26.2 merge) fix: `fcli fod microservice create`: Re-add non-microservice application guard (lost in Aviator 26.2 merge) --- .../dast/FoDScanDastAutomatedHelper.java | 19 ++++++++---- .../cmd/FoDDastAutomatedScanStartCommand.java | 31 +++++++++++++++++-- .../cli/cmd/FoDMicroserviceCreateCommand.java | 5 +++ .../cli/fod/i18n/FoDMessages.properties | 1 + 4 files changed, 47 insertions(+), 9 deletions(-) diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/scan/helper/dast/FoDScanDastAutomatedHelper.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/scan/helper/dast/FoDScanDastAutomatedHelper.java index 731017cb6e2..e1b16c32489 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/scan/helper/dast/FoDScanDastAutomatedHelper.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/scan/helper/dast/FoDScanDastAutomatedHelper.java @@ -80,7 +80,11 @@ public static final FoDScanDescriptor handleInProgressScan(UnirestInstance unire .queryString("fields", "scanId,scanType,analysisStatusType") .asObject(JsonNode.class).getBody(); JsonNode itemsNode = response.path("items"); - if (!itemsNode.isArray() || itemsNode.isEmpty()) continue; + if (!itemsNode.isArray() || itemsNode.isEmpty()) { + // No scans exist for this release yet; nothing to handle. + progressWriter.writeProgress("Status: No previous scans found"); + return null; + } boolean foundActive = false; for (JsonNode node : itemsNode) { @@ -107,10 +111,13 @@ public static final FoDScanDescriptor handleInProgressScan(UnirestInstance unire throw new FcliSimpleException("Unable to start Dynamic scan after " + maxAttempts + " attempts. Please check the UI and try again."); } - public static final FoDScanDescriptor startScan(UnirestInstance unirest, FoDReleaseDescriptor releaseDescriptor) { - JsonNode response = unirest.post(FoDUrls.DAST_AUTOMATED_SCANS + "/start-scan") - .routeParam("relId", releaseDescriptor.getReleaseId()) - .asObject(JsonNode.class).getBody(); + public static final FoDScanDescriptor startScan(UnirestInstance unirest, String networkName, FoDReleaseDescriptor releaseDescriptor) { + var request = unirest.post(FoDUrls.DAST_AUTOMATED_SCANS + "/start-scan") + .routeParam("relId", releaseDescriptor.getReleaseId()); + if (networkName != null && !networkName.isBlank()) { + request = request.queryString("networkName", networkName); + } + JsonNode response = request.asObject(JsonNode.class).getBody(); FoDStartScanResponse startScanResponse = JsonHelper.treeToValue(response, FoDStartScanResponse.class); if (startScanResponse == null || startScanResponse.getScanId() <= 0) { throw new FcliSimpleException("Unable to retrieve scan id from response when starting Dynamic scan."); @@ -151,7 +158,7 @@ private static FoDScanDescriptor handleActiveScan( Thread.sleep(waitMillis); break; case DoNotStartScan: - progressWriter.writeProgress("Status: A scan is running %s, no new scan will be started", scanId); + progressWriter.writeProgress("Status: A scan with id %s is %s, no new scan will be started", scanId, status); JsonNode scan = objectMapper.createObjectNode() .put("scanId", scanId) .put("scanType", FoDScanType.Dynamic.name()) diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/dast_scan/cli/cmd/FoDDastAutomatedScanStartCommand.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/dast_scan/cli/cmd/FoDDastAutomatedScanStartCommand.java index c68709a4f1e..6701c7d0b4f 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/dast_scan/cli/cmd/FoDDastAutomatedScanStartCommand.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/dast_scan/cli/cmd/FoDDastAutomatedScanStartCommand.java @@ -12,8 +12,10 @@ */ package com.fortify.cli.fod.dast_scan.cli.cmd; +import com.fortify.cli.common.exception.FcliSimpleException; import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; import com.fortify.cli.common.progress.cli.mixin.ProgressWriterFactoryMixin; +import com.fortify.cli.common.rest.unirest.UnexpectedHttpResponseException; import com.fortify.cli.fod._common.scan.cli.cmd.AbstractFoDScanStartCommand; import com.fortify.cli.fod._common.scan.cli.mixin.FoDInProgressScanActionTypeMixins; import com.fortify.cli.fod._common.scan.helper.FoDScanDescriptor; @@ -36,6 +38,8 @@ public class FoDDastAutomatedScanStartCommand extends AbstractFoDScanStartComman private Integer waitInterval; @Option(names="--max-attempts", descriptionKey = "fcli.fod.scan.max-attempts", defaultValue = "30", required = false) private Integer maxAttempts; + @Option(names = {"--vpn"}, descriptionKey = "fcli.fod.dast-scan.start.vpn") + private String fodConnectNetwork; @Mixin private ProgressWriterFactoryMixin progressWriterFactory; @@ -48,13 +52,19 @@ protected FoDScanDescriptor startScan(UnirestInstance unirest, FoDReleaseDescrip try (var progressWriter = progressWriterFactory.create()) { // get current setup to ensure the scan has been configured - FoDScanDastAutomatedHelper.getSetupDescriptor(unirest, relId); + var setup = FoDScanDastAutomatedHelper.getSetupDescriptor(unirest, relId); + if (setup == null) { + throw new FcliSimpleException("DAST Automated scan is not configured for release '" + + releaseDescriptor.getReleaseName() + + "'. Please run one of the 'fod dast-scan setup-xxx' commands to configure the scan before starting."); + } - // check if scan is already in progress + // Handle any in-progress or active scans according to configured action. FoDScanDescriptor scan = FoDScanDastAutomatedHelper.handleInProgressScan(unirest, releaseDescriptor, inProgressScanActionType.getInProgressScanActionType(), progressWriter, maxAttempts, waitInterval); + // If the action was to not start a new scan, return current in-progress descriptor. if (scan != null && scan.getAnalysisStatusType().equals("In_Progress")) { if (inProgressScanActionType.getInProgressScanActionType() == FoDEnums.InProgressScanActionType.DoNotStartScan) { scanAction = "NOT_STARTED_SCAN_IN_PROGRESS"; @@ -62,7 +72,22 @@ protected FoDScanDescriptor startScan(UnirestInstance unirest, FoDReleaseDescrip } } - return FoDScanDastAutomatedHelper.startScan(unirest, releaseDescriptor); + try { + return FoDScanDastAutomatedHelper.startScan(unirest, fodConnectNetwork, releaseDescriptor); + } catch (UnexpectedHttpResponseException e) { + // If FoD reports active scan race, resolve and return that scan instead of failing. + if (e.getStatus() == 422 && (e.getMessage().contains("dynamic scan is currently in progress") + || e.getMessage().contains("errorCode: -10"))) { + FoDScanDescriptor running = FoDScanDastAutomatedHelper.handleInProgressScan(unirest, releaseDescriptor, + FoDEnums.InProgressScanActionType.DoNotStartScan, progressWriter, maxAttempts, + waitInterval); + if (running != null) { + scanAction = "NOT_STARTED_SCAN_IN_PROGRESS"; + return running; + } + } + throw e; + } } } diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/microservice/cli/cmd/FoDMicroserviceCreateCommand.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/microservice/cli/cmd/FoDMicroserviceCreateCommand.java index 1a6f78c5023..14d526380d8 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/microservice/cli/cmd/FoDMicroserviceCreateCommand.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/microservice/cli/cmd/FoDMicroserviceCreateCommand.java @@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fortify.cli.common.cli.mixin.CommonOptionMixins; +import com.fortify.cli.common.exception.FcliSimpleException; import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; import com.fortify.cli.common.output.transform.IActionCommandResultSupplier; import com.fortify.cli.fod._common.cli.mixin.FoDDelimiterMixin; @@ -52,6 +53,10 @@ public JsonNode getJsonNode(UnirestInstance unirest) { } FoDAppDescriptor appDescriptor = qualifiedMicroserviceNameResolver.getAppDescriptor(unirest, true); FoDQualifiedMicroserviceNameDescriptor qualifiedMicroserviceNameDescriptor = qualifiedMicroserviceNameResolver.getQualifiedMicroserviceNameDescriptor(); + if (!appDescriptor.isHasMicroservices()) { + throw new FcliSimpleException("Cannot create microservice for non-microservice application " + + appDescriptor.getApplicationName()); + } FoDMicroserviceUpdateRequest msCreateRequest = FoDMicroserviceUpdateRequest.builder() .microserviceName(qualifiedMicroserviceNameDescriptor.getMicroserviceName()) .attributes(new FoDAttributeDefinitionHelper(unirest).buildAttributesNode(FoDEnums.AttributeTypes.Microservice, diff --git a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties index 6248bee37db..a48b6188128 100644 --- a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties +++ b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties @@ -639,6 +639,7 @@ fcli.fod.dast-scan.start.usage.description.0 = This command is intended for DAST fcli.fod.dast-scan.start.usage.description.1 = The scan will need to have been previously setup using the FoD UI or one of the \ 'fod dast-scan setup-xxx' commands. fcli.fod.dast-scan.start.validate-entitlement = Validate if an entitlement has been set and is still valid. +fcli.fod.dast-scan.start.vpn = Fortify Connect network name to use for site-to-site VPN. fcli.fod.dast-scan.start-legacy.usage.header = (LEGACY) Start a new DAST scan. fcli.fod.dast-scan.start-legacy.usage.description.0 = This command is not fully implemented and is intended for legacy DAST scanning (not DAST Automated). \ It can only be used for starting a configured Dynamic scan and does not support file uploads (i.e. API definitions \ From f4f5bf1498a4ff79ba4c8db38a317508373f43ec Mon Sep 17 00:00:00 2001 From: Madhur Jain <87946372+jmadhur87@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:00:32 +0530 Subject: [PATCH 04/37] chore: Add issue get command for FoD and SSC (#978) feat: `fcli fod issue get`: New command for retrieving issue data for a single issue feat: `fcli ssc issue get`: New command for retrieving issue data for a single issue --- .../fod/issue/cli/cmd/FoDIssueCommands.java | 1 + .../fod/issue/cli/cmd/FoDIssueGetCommand.java | 83 +++++++++++++++++++ .../cli/fod/issue/helper/FoDIssueHelper.java | 4 +- .../cli/fod/i18n/FoDMessages.properties | 10 ++- .../ssc/issue/cli/cmd/SSCIssueCommands.java | 1 + .../ssc/issue/cli/cmd/SSCIssueGetCommand.java | 55 ++++++++++++ .../issue/cli/mixin/SSCIssueIncludeMixin.java | 2 +- .../helper/SSCIssueEmbedderSupplier.java | 44 ++++++---- .../cli/ssc/i18n/SSCMessages.properties | 10 ++- 9 files changed, 186 insertions(+), 24 deletions(-) create mode 100644 fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/cli/cmd/FoDIssueGetCommand.java create mode 100644 fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/cmd/SSCIssueGetCommand.java diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/cli/cmd/FoDIssueCommands.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/cli/cmd/FoDIssueCommands.java index 3fa8be83e6d..cda827f0873 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/cli/cmd/FoDIssueCommands.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/cli/cmd/FoDIssueCommands.java @@ -19,6 +19,7 @@ @CommandLine.Command(name = "issue", subcommands = { FoDIssueListCommand.class, + FoDIssueGetCommand.class, FoDIssueUpdateCommand.class, } ) diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/cli/cmd/FoDIssueGetCommand.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/cli/cmd/FoDIssueGetCommand.java new file mode 100644 index 00000000000..a9d1a557c3b --- /dev/null +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/cli/cmd/FoDIssueGetCommand.java @@ -0,0 +1,83 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.fod.issue.cli.cmd; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.common.exception.FcliSimpleException; +import com.fortify.cli.common.exception.FcliTechnicalException; +import com.fortify.cli.common.json.producer.IObjectNodeProducer; +import com.fortify.cli.common.json.producer.ObjectNodeProducerApplyFrom; +import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; +import com.fortify.cli.fod._common.cli.mixin.FoDDelimiterMixin; +import com.fortify.cli.fod._common.output.cli.cmd.AbstractFoDOutputCommand; +import com.fortify.cli.fod._common.rest.FoDUrls; +import com.fortify.cli.fod._common.rest.helper.FoDInputTransformer; +import com.fortify.cli.fod.issue.cli.mixin.FoDIssueEmbedMixin; +import com.fortify.cli.fod.release.cli.mixin.FoDReleaseByQualifiedNameOrIdResolverMixin; +import com.fortify.cli.fod.release.helper.FoDReleaseDescriptor; + +import kong.unirest.HttpRequest; +import kong.unirest.UnirestInstance; +import lombok.Getter; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Parameters; + +@Command(name = OutputHelperMixins.Get.CMD_NAME) +public class FoDIssueGetCommand extends AbstractFoDOutputCommand { + @Getter @Mixin private OutputHelperMixins.Get outputHelper; + @Mixin private FoDDelimiterMixin delimiterMixin; // Is automatically injected in resolver mixins + @Mixin private FoDReleaseByQualifiedNameOrIdResolverMixin.RequiredOption releaseResolver; + @Parameters(index = "0", arity = "1", descriptionKey = "fcli.fod.issue.get.vulnId") + private String vulnId; + @Mixin private FoDIssueEmbedMixin embedMixin; + + @Override + protected IObjectNodeProducer getObjectNodeProducer(UnirestInstance unirest) { + FoDReleaseDescriptor releaseDescriptor = releaseResolver.getReleaseDescriptor(unirest); + ObjectNode issue = findIssue(unirest, releaseDescriptor); + issue.put("releaseName", releaseDescriptor.getReleaseName()); + return simpleObjectNodeProducerBuilder(ObjectNodeProducerApplyFrom.SPEC) + .source(issue) + .build(); + } + + private ObjectNode findIssue(UnirestInstance unirest, FoDReleaseDescriptor releaseDescriptor) { + String releaseId = releaseDescriptor.getReleaseId().toString(); + HttpRequest request = unirest.get(FoDUrls.VULNERABILITIES) + .routeParam("relId", releaseId) + .queryString("filters", "vulnId:" + vulnId) + .queryString("includeFixed", "true") + .queryString("includeSuppressed", "true") + .queryString("limit", "2"); + JsonNode body = request.asObject(JsonNode.class).getBody(); + JsonNode items = FoDInputTransformer.getItems(body); + if ( items==null || !items.isArray() || items.isEmpty() ) { + throw new FcliSimpleException(String.format("No vulnerability found for vulnId '%s' in release '%s'", vulnId, releaseDescriptor.getReleaseName())); + } + if ( items.size()>1 ) { + throw new FcliSimpleException(String.format("Multiple vulnerabilities found for vulnId '%s'; please check your input", vulnId)); + } + JsonNode issue = items.get(0); + if ( !(issue instanceof ObjectNode issueObject) ) { + throw new FcliTechnicalException(String.format("Unexpected response for vulnId '%s'; expected a JSON object", vulnId)); + } + return issueObject; + } + + @Override + public boolean isSingular() { + return true; + } +} \ No newline at end of file diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/helper/FoDIssueHelper.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/helper/FoDIssueHelper.java index cef7e049915..a44b2941c71 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/helper/FoDIssueHelper.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/helper/FoDIssueHelper.java @@ -65,7 +65,7 @@ public static IssueAggregationData forSingleRelease(ObjectNode issue) { .releaseNames(Set.of(releaseName)) .releaseIds(Set.of(releaseId)) .ids(Set.of(id)) - .vulnIds(Set.of(vulnId)) + .vulnIds(vulnId!=null ? Set.of(vulnId) : Collections.emptySet()) .build(); } @@ -92,7 +92,7 @@ public String getReleaseIdsString() { } public String getIdsString() { - return asString(ids); + return asString(ids); } private String asString(Set values) { diff --git a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties index a48b6188128..c4ba5bd2503 100644 --- a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties +++ b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties @@ -932,9 +932,9 @@ fcli.fod.issue.list.usage.description = This command allows for listing FoD issu recommended to use server-side filtering, via use of the --filters-param or --query options. \ For example, if you are only interested in issues with a specific severity, you \ can use a query like --filters-param "severityString:Critical" or --query "severityString='Critical'". -fcli.fod.issue.list.output.table.header.visibilityMarker = -fcli.fod.issue.list.output.table.header.foundInReleases = Releases -fcli.fod.issue.list.output.table.header.foundInReleasesString = Releases +fcli.fod.issue.output.table.header.visibilityMarker = +fcli.fod.issue.output.table.header.foundInReleases = Releases +fcli.fod.issue.output.table.header.foundInReleasesString = Releases fcli.fod.issue.embed = Embed extra issue data. Due to FoD rate limits, this may significantly \ affect performance. Allowed values: ${COMPLETION-CANDIDATES}. \ Using the --output option, this extra data can be included in the output. Using the --query option, \ @@ -945,6 +945,9 @@ fcli.fod.issue.list.includeIssue = By default, only visible issues will be retur for example `--include visible,fixed` (to return both visible and fixed issues) or `--include \ fixed` (to return only fixed issues). Allowed values: ${COMPLETION-CANDIDATES}. fcli.fod.issue.list.aggregate = Include aggregation data. +fcli.fod.issue.get.usage.header = Get vulnerability details. +fcli.fod.issue.get.usage.description = Get detailed data for a single FoD vulnerability in a given release. +fcli.fod.issue.get.vulnId = Issue vulnerability id. fcli.fod.issue.update.usage.header = Bulk update issues (vulnerabilities). fcli.fod.issue.update.usage.description = This command allows for updating the audit information \ for multiple issues (vulnerabilities). Note: for --vuln-ids/--issue-ids, you can use either the numeric Id as shown in the FoD UI, \ @@ -1091,6 +1094,7 @@ fcli.fod.rest.lookup.output.table.args = group,text,value fcli.fod.report.output.table.args = reportId,reportName,reportStatusType,reportType fcli.fod.report.report-template.output.table.args = value,text,group fcli.fod.issue.list.output.table.args = instanceId,visibilityMarker,severityString,category,location,foundInReleasesString +fcli.fod.issue.get.output.table.args = instanceId,severityString,category,primaryLocation,releaseName fcli.fod.issue.update.output.table.args = totalCount,updateCount,skippedCount,errorCount fcli.fod.attribute.output.table.args = id,name,attributeType,attributeDataType,isRequired,isRestricted fcli.fod.aviator.apply-remediations.output.table.args = releaseId,totalRemediation,appliedRemediation,skippedRemediation,skippedReasons,__action__ diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/cmd/SSCIssueCommands.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/cmd/SSCIssueCommands.java index a3d1b5e40bf..54b55eea3d4 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/cmd/SSCIssueCommands.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/cmd/SSCIssueCommands.java @@ -32,6 +32,7 @@ SSCIssueGroupGetCommand.class, SSCIssueGroupListCommand.class, SSCIssueCountCommand.class, + SSCIssueGetCommand.class, SSCIssueListCommand.class, SSCIssueUpdateCommand.class, } diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/cmd/SSCIssueGetCommand.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/cmd/SSCIssueGetCommand.java new file mode 100644 index 00000000000..863b3aea270 --- /dev/null +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/cmd/SSCIssueGetCommand.java @@ -0,0 +1,55 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.ssc.issue.cli.cmd; + +import com.fortify.cli.common.cli.util.EnvSuffix; +import com.fortify.cli.common.json.producer.IObjectNodeProducer; +import com.fortify.cli.common.json.producer.ObjectNodeProducerApplyFrom; +import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; +import com.fortify.cli.ssc._common.output.cli.cmd.AbstractSSCOutputCommand; +import com.fortify.cli.ssc._common.rest.ssc.SSCUrls; +import com.fortify.cli.ssc.appversion.cli.mixin.SSCAppVersionResolverMixin; +import com.fortify.cli.ssc.issue.cli.mixin.SSCIssueBulkEmbedMixin; + +import kong.unirest.HttpRequest; +import kong.unirest.UnirestInstance; +import lombok.Getter; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Parameters; + +@Command(name = OutputHelperMixins.Get.CMD_NAME) +public class SSCIssueGetCommand extends AbstractSSCOutputCommand { + @Getter @Mixin private OutputHelperMixins.Get outputHelper; + @Mixin private SSCAppVersionResolverMixin.RequiredOption parentResolver; + @EnvSuffix("ISSUE_ID") @Parameters(index = "0", arity = "1", descriptionKey = "fcli.ssc.issue.get.id") + private String id; + @Mixin private SSCIssueBulkEmbedMixin bulkEmbedMixin; + + @Override + protected IObjectNodeProducer getObjectNodeProducer(UnirestInstance unirest) { + String appVersionId = parentResolver.getAppVersionId(unirest); + return requestObjectNodeProducerBuilder(ObjectNodeProducerApplyFrom.SPEC) + .baseRequest(getBaseRequest(unirest, appVersionId)) + .build(); + } + + private HttpRequest getBaseRequest(UnirestInstance unirest, String appVersionId) { + return unirest.get(SSCUrls.PROJECT_VERSION_ISSUE(appVersionId, id)); + } + + @Override + public boolean isSingular() { + return true; + } +} diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/mixin/SSCIssueIncludeMixin.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/mixin/SSCIssueIncludeMixin.java index 2a7a574c158..fcf0f497b2d 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/mixin/SSCIssueIncludeMixin.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/mixin/SSCIssueIncludeMixin.java @@ -31,7 +31,7 @@ public class SSCIssueIncludeMixin implements IHttpRequestUpdater, IRecordTransformer { @DisableTest(TestType.MULTI_OPT_PLURAL_NAME) - @Option(names = {"--include", "-i"}, split = ",", defaultValue = "visible", descriptionKey = "fcli.ssc.issue.list.includeIssue", paramLabel="") + @Option(names = {"--include", "-i"}, split = ",", defaultValue = "visible", descriptionKey = "fcli.ssc.issue.includeIssue", paramLabel="") private Set includes; public HttpRequest updateRequest(HttpRequest request) { diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/helper/SSCIssueEmbedderSupplier.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/helper/SSCIssueEmbedderSupplier.java index a7f02ed3fa1..0ca97ec3640 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/helper/SSCIssueEmbedderSupplier.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/helper/SSCIssueEmbedderSupplier.java @@ -39,49 +39,61 @@ public ISSCEntityEmbedder createEntityEmbedder() { } private static abstract class AbstractSSCIssueEmbedder implements ISSCEntityEmbedder { + protected abstract HttpRequest getBaseRequest(UnirestInstance unirest); + + protected abstract void process(ObjectNode record, JsonNode response); + + // Whether the embedder's endpoint supports the 'limit' parameter to retrieve all records + protected boolean supportsLimit() { return true; } + @Override - public void addEmbedRequests(SSCBulkRequestBuilder builder, UnirestInstance unirest, JsonNode record) { + public final void addEmbedRequests(SSCBulkRequestBuilder builder, UnirestInstance unirest, JsonNode record) { var id = record.get("id").asText(); - builder.request(getBaseRequest(unirest) - .routeParam("id", id) - .queryString("limit", "-1"), - response->process((ObjectNode)record, SSCInputTransformer.getDataOrSelf(response))); + HttpRequest request = getBaseRequest(unirest).routeParam("id", id); + if ( supportsLimit() ) { request = request.queryString("limit", "-1"); } + builder.request( + request, + response -> process((ObjectNode) record, SSCInputTransformer.getDataOrSelf(response)) + ); } - - protected abstract HttpRequest getBaseRequest(UnirestInstance unirest); - protected abstract void process(ObjectNode record, JsonNode response); } - + private static final class SSCIssueDetailsEmbedder extends AbstractSSCIssueEmbedder { + @Override + protected boolean supportsLimit() { return false; } + @Override protected HttpRequest getBaseRequest(UnirestInstance unirest) { return unirest.get("/api/v1/issueDetails/{id}"); } + @Override protected void process(ObjectNode record, JsonNode response) { - record.set("details", response); + record.set("details", response); } } - + private static final class SSCIssueAuditHistoryEmbedder extends AbstractSSCIssueEmbedder { @Override protected HttpRequest getBaseRequest(UnirestInstance unirest) { - return unirest.get("/api/v1/issues/{id}/auditHistory?limit=-1"); + return unirest.get("/api/v1/issues/{id}/auditHistory"); } + @Override protected void process(ObjectNode record, JsonNode response) { - record.set("auditHistory", response); + record.set("auditHistory", response); } } - + private static final class SSCIssueCommentsEmbedder extends AbstractSSCIssueEmbedder { @Override protected HttpRequest getBaseRequest(UnirestInstance unirest) { - return unirest.get("/api/v1/issues/{id}/comments?limit=-1"); + return unirest.get("/api/v1/issues/{id}/comments"); } + @Override protected void process(ObjectNode record, JsonNode response) { - record.set("comments", response); + record.set("comments", response); } } } diff --git a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties index d2dd0ebe86a..2e8fb3a297a 100644 --- a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties +++ b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties @@ -514,13 +514,16 @@ fcli.ssc.issue.list.usage.description = This command allows for listing SSC vuln more immediate output. fcli.ssc.issue.list.output.table.header.visibilityMarker = fcli.ssc.issue.list.output.table.header.friority = Priority +fcli.ssc.issue.get.usage.header = Get vulnerability details. +fcli.ssc.issue.get.usage.description = Get detailed data for a single SSC vulnerability in a given application version. +fcli.ssc.issue.get.id = Issue id. fcli.ssc.issue.list.filter = Filter issues using the given (friendly or technical) filter. \ See 'fcli ssc issue list-filters' for allowed values. -fcli.ssc.issue.list.embed = Embed extra application version data. Allowed values: ${COMPLETION-CANDIDATES}. \ +fcli.ssc.issue.embed = Embed extra application version data. Allowed values: ${COMPLETION-CANDIDATES}. \ Using the --output option, this extra data can be included in the output. Using the --query option, \ this extra data can be queried upon. To get an understanding of the structure and contents of the \ embedded data, use the --output json or --output yaml options. -fcli.ssc.issue.list.includeIssue = By default, only visible issues will be returned. This option \ +fcli.ssc.issue.includeIssue = By default, only visible issues will be returned. This option \ accepts a comma-separated list to allow (also) removed, suppressed and/or hidden issues to be returned, \ for example `--include visible,removed` (to return both visible and removed issues) or `--include \ removed` (to return only removed issues). Allowed values: ${COMPLETION-CANDIDATES}. @@ -715,6 +718,9 @@ fcli.ssc.attribute.definition.output.table.args = id,category,guid,name,type,req fcli.ssc.aviator.output.table.args = id,application.name,name,artifactId fcli.ssc.custom-tag.output.table.args = guid,name,valueType fcli.ssc.issue.count.output.table.args = cleanName,totalCount,auditedCount +fcli.ssc.issue.get.output.table.args = id,visibilityMarker,friority,location,issueName +fcli.ssc.issue.get.output.table.header.visibilityMarker = +fcli.ssc.issue.get.output.table.header.friority = Priority fcli.ssc.issue.list.output.table.args = id,visibilityMarker,friority,location,issueName fcli.ssc.issue.filter-set.output.table.args = guid,title,defaultFilterSet fcli.ssc.issue.group.output.table.args = guid,displayName,entityType From 65ddc653789849de6d5703c5b40a5058df95c1c7 Mon Sep 17 00:00:00 2001 From: "Sangamesh V." <60685551+SangameshV@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:36:14 +0530 Subject: [PATCH 05/37] chore: Add `--server-name` option on stdio MCP server (#1077) feat: `fcli ai-assist mcp start-stdio`: Add `--server-name` option to configure custom MCP server name, defaulting to either `fcli-` or `fcli` (depending on whether `--module` is specified) --- .../mcp/cli/cmd/AiAssistMCPStartStdioCommand.java | 10 +++++++++- .../cli/ai_assist/i18n/AiAssistMessages.properties | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/cli/cmd/AiAssistMCPStartStdioCommand.java b/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/cli/cmd/AiAssistMCPStartStdioCommand.java index 0301fcf0a59..8c82de69216 100644 --- a/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/cli/cmd/AiAssistMCPStartStdioCommand.java +++ b/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/cli/cmd/AiAssistMCPStartStdioCommand.java @@ -93,6 +93,7 @@ public class AiAssistMCPStartStdioCommand extends AbstractRunnableCommand implem @Option(names={"--module", "-m"}, required = false) private McpModule module; @DisableTest(TestType.MULTI_OPT_PLURAL_NAME) @Option(names={"--import"}, split=",") private List importFiles; + @Option(names={"--server-name"}) private String serverName; @Option(names={"--work-threads"}, defaultValue="10") private int workThreads; @Option(names={"--progress-threads"}, defaultValue="4") private int progressThreads; @Option(names={"--job-safe-return"}, defaultValue="25s") private String jobSafeReturnPeriod; @@ -111,6 +112,13 @@ public Integer call() throws Exception { if (module == null && (importFiles == null || importFiles.isEmpty())) { throw new FcliSimpleException("At least one of --module or --import must be specified"); } + if (serverName == null) { + if (module != null) { + serverName = "fcli-" + module.toString(); + } else { + serverName = "fcli"; + } + } var rawOut = StdioHelper.getRawOut(); var rawErr = StdioHelper.getRawErr(); // Redirect progress output to stderr to prevent progress messages @@ -147,7 +155,7 @@ public Integer call() throws Exception { // Use rawOut to bypass the delegation/masking stack, ensuring // MCP JSON-RPC responses are never corrupted by masking var serverBuilder = McpServer.sync(new StdioServerTransportProvider(new JacksonMcpJsonMapper(objectMapper), wrappedIn, rawOut)) - .serverInfo("fcli", FcliBuildProperties.INSTANCE.getFcliVersion()) + .serverInfo(serverName, FcliBuildProperties.INSTANCE.getFcliVersion()) .requestTimeout(Duration.ofSeconds(120)) .instructions(""" - For tools that accept a --*-session option and user hasn't asked for a specific \ diff --git a/fcli-core/fcli-ai-assist/src/main/resources/com/fortify/cli/ai_assist/i18n/AiAssistMessages.properties b/fcli-core/fcli-ai-assist/src/main/resources/com/fortify/cli/ai_assist/i18n/AiAssistMessages.properties index 30b5794df96..0d50f0afd0d 100644 --- a/fcli-core/fcli-ai-assist/src/main/resources/com/fortify/cli/ai_assist/i18n/AiAssistMessages.properties +++ b/fcli-core/fcli-ai-assist/src/main/resources/com/fortify/cli/ai_assist/i18n/AiAssistMessages.properties @@ -81,6 +81,7 @@ fcli.ai-assist.mcp.start-stdio.progress-threads = Number of threads used for upd fcli.ai-assist.mcp.start-stdio.job-safe-return = Maximum time to wait synchronously for a job result before returning an in_progress placeholder. Specify duration like 25s, 2m, 1h. fcli.ai-assist.mcp.start-stdio.progress-interval = Interval between internal progress counter updates for long-running jobs. Specify duration (e.g. 500ms, 1s, 2s). fcli.ai-assist.mcp.start-stdio.async-bg-threads = Number of background threads for running async streaming jobs. Default: 2. +fcli.ai-assist.mcp.start-stdio.server-name = Custom name for this MCP server instance. If not specified, defaults to 'fcli-' when --module is provided, or 'fcli-imported-functions' when only --import is provided. fcli.ai-assist.mcp.start-http.usage.header = Start fcli HTTP MCP server. fcli.ai-assist.mcp.start-http.usage.description = Start an HTTP MCP server exposing only exported functions from imported action YAML files defined in a config file. Generate a sample config file with 'fcli ai-assist mcp create-http-config --type ' and customize the generated YAML for your environment. The server listens for MCP POST requests on the /mcp endpoint. Each request must include the product-specific auth header as semicolon-separated key=value pairs; escape literal '\\', ';', or '=' characters as '\\\\', '\\;', or '\\='.\ From 1c609217e2b905e46dcdc64db7a23052e39a274d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:00:27 +0000 Subject: [PATCH 06/37] chore(rel/v3.x): release 3.24.0 --- .release-please-manifest.json | 2 +- CHANGELOG.md | 25 +++++++++++++++++++++++++ version.txt | 2 +- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 1f12ca48152..6973dab8638 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "3.23.3" + ".": "3.24.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 17879e3a19a..53e8bbefaed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [3.24.0](https://github.com/fortify/fcli/compare/v3.23.3...v3.24.0) (2026-08-21) + + +### Features + +* `fcli ai-assist mcp start-stdio`: Add `--server-name` option to configure custom MCP server name, defaulting to either `fcli-` or `fcli` (depending on whether `--module` is specified) ([65ddc65](https://github.com/fortify/fcli/commit/65ddc653789849de6d5703c5b40a5058df95c1c7)) +* `fcli aviator ssc apply-remediations`: Add `--source-encodings` option for source file decoding and encoding ([e63c814](https://github.com/fortify/fcli/commit/e63c814e531c5aab8748a145dff4000c0c21913e)) +* `fcli aviator ssc audit`: Add `--source-encodings` option for source decoding and audit skip reporting ([e63c814](https://github.com/fortify/fcli/commit/e63c814e531c5aab8748a145dff4000c0c21913e)) +* `fcli fod aviator apply-remediations`: Add `--source-encodings` option for source file decoding and encoding ([e63c814](https://github.com/fortify/fcli/commit/e63c814e531c5aab8748a145dff4000c0c21913e)) +* `fcli fod issue get`: New command for retrieving issue data for a single issue ([f4f5bf1](https://github.com/fortify/fcli/commit/f4f5bf1498a4ff79ba4c8db38a317508373f43ec)) +* `fcli fod session login`: Add `--code` and `--totp` options to support MFA-based logins (resolves [#1059](https://github.com/fortify/fcli/issues/1059)) ([cf882bb](https://github.com/fortify/fcli/commit/cf882bb41eafa92f66e570facad1fb0a008a78f5)) +* `fcli ssc issue get`: New command for retrieving issue data for a single issue ([f4f5bf1](https://github.com/fortify/fcli/commit/f4f5bf1498a4ff79ba4c8db38a317508373f43ec)) + + +### Bug Fixes + +* `fcli aviator ssc apply-remediations`: apply source edits using the encoding declared in `audit.fvdl` instead of assuming UTF-8 ([71919f7](https://github.com/fortify/fcli/commit/71919f76a58fa21ba029b03659ee7202c010e16f)) +* `fcli aviator ssc apply-remediations`: Atomic Remediation with Exception Handling and Compliance Reporting ([92bcbe1](https://github.com/fortify/fcli/commit/92bcbe1aa035ec29a807075333ce3603545ca55d)) +* `fcli aviator ssc audit`: Upload only issues written in the current audit run ([#1063](https://github.com/fortify/fcli/issues/1063)) ([97e587a](https://github.com/fortify/fcli/commit/97e587ab0cd730b3fff76c8d4f1963cb63e5f2d0)) +* `fcli fod aviator apply-remediations`: apply source edits using the encoding declared in `audit.fvdl` instead of assuming UTF-8 ([71919f7](https://github.com/fortify/fcli/commit/71919f76a58fa21ba029b03659ee7202c010e16f)) +* `fcli fod aviator apply-remediations`: Atomic Remediation with Exception Handling and Compliance Reporting ([92bcbe1](https://github.com/fortify/fcli/commit/92bcbe1aa035ec29a807075333ce3603545ca55d)) +* `fcli fod dast-scan start`: Restore `--vpn` support and 422 active-scan fallback (lost in Aviator 26.2 merge) ([fbf4f7b](https://github.com/fortify/fcli/commit/fbf4f7bcbd20f24e56445a2b70adc056d7e115fc)) +* `fcli fod dast-scan start`: Restore fix to allow DAST scan to start if no prior scans exist (lost in Aviator 26.2 merge) (fixes [#1068](https://github.com/fortify/fcli/issues/1068)) ([fbf4f7b](https://github.com/fortify/fcli/commit/fbf4f7bcbd20f24e56445a2b70adc056d7e115fc)) +* `fcli fod microservice create`: Re-add non-microservice application guard (lost in Aviator 26.2 merge) ([fbf4f7b](https://github.com/fortify/fcli/commit/fbf4f7bcbd20f24e56445a2b70adc056d7e115fc)) + ## [3.23.3](https://github.com/fortify/fcli/compare/v3.23.2...v3.23.3) (2026-07-15) diff --git a/version.txt b/version.txt index c873913dd16..954e2288218 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.23.3 +3.24.0 From b1200e2d5b3b3b510a33485628358e216470134c Mon Sep 17 00:00:00 2001 From: Ruud Senden <8635138+rsenden@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:19:47 +0200 Subject: [PATCH 07/37] chore: Update int to Long_Int (#1081) --- .../cli/cmd/AbstractSSCIssueTemplateUpdateCommand.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue_template/cli/cmd/AbstractSSCIssueTemplateUpdateCommand.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue_template/cli/cmd/AbstractSSCIssueTemplateUpdateCommand.java index 1ec99fa9144..ac8a70ae2bb 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue_template/cli/cmd/AbstractSSCIssueTemplateUpdateCommand.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue_template/cli/cmd/AbstractSSCIssueTemplateUpdateCommand.java @@ -16,7 +16,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.IntNode; +import com.fasterxml.jackson.databind.node.LongNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fortify.cli.common.json.JsonHelper; import com.fortify.cli.common.output.transform.IActionCommandResultSupplier; @@ -63,7 +63,7 @@ protected ArrayNode getCustomTagIds(UnirestInstance unirest, SSCIssueTemplateDes currentTags, addTagsMixin.getTagSpecs(), rmTagsMixin.getTagSpecs() - ).map(tag -> new IntNode(Integer.valueOf(tag.getId()))).collect(JsonHelper.arrayNodeCollector()); + ).map(tag -> new LongNode(Long.valueOf(tag.getId()))).collect(JsonHelper.arrayNodeCollector()); } @Override From 840ee90beed814507f06c01c0e456f1b610f4234 Mon Sep 17 00:00:00 2001 From: Kireeti Varma <81861747+kireetivar@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:00:06 +0530 Subject: [PATCH 08/37] chore: Add `--force-reaudit` option, fix remediation handling (#1082) feat: `fcli aviator ssc audit`: Add `--force-reaudit` to re-audit Aviator-processed issues without overwriting human triage fix: `fcli aviator ssc apply-remediations`: Skip remediations when source context matches multiple locations --- .../fortify/cli/aviator/audit/AuditFPR.java | 41 +-- .../cli/aviator/audit/IssueAuditor.java | 203 ++++++++++---- .../aviator/audit/model/AuditFprOptions.java | 1 + .../fortify/cli/aviator/fpr/FPRProcessor.java | 13 +- .../cli/aviator/fpr/model/AuditIssue.java | 1 + .../aviator/fpr/processor/AuditProcessor.java | 19 ++ .../fpr/processor/RemediationProcessor.java | 77 +++++- .../fortify/cli/aviator/util/Constants.java | 16 +- .../aviator/util/FuzzyContextSearcher.java | 86 +++--- .../fortify/cli/aviator/util/StringUtil.java | 11 + .../cli/aviator/audit/IssueAuditorTest.java | 227 +++++++++++++++- .../cli/aviator/fpr/FPRProcessorTest.java | 14 + .../AuditProcessorLastTagUsernamesTest.java | 129 +++++++++ .../processor/RemediationProcessorTest.java | 145 ++++++++++ .../util/FuzzyContextSearcherTest.java | 51 ++++ .../ssc/cli/cmd/AviatorSSCAuditCommand.java | 7 +- .../ssc/helper/AviatorSSCAuditHelper.java | 211 +++++++++------ .../aviator/i18n/AviatorMessages.properties | 1 + .../cli/cmd/AviatorSSCAuditCommandTest.java | 9 + .../ssc/helper/AviatorSSCAuditHelperTest.java | 252 ++++++++++++++++++ 20 files changed, 1280 insertions(+), 234 deletions(-) create mode 100644 fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/processor/AuditProcessorLastTagUsernamesTest.java create mode 100644 fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessorTest.java create mode 100644 fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelperTest.java diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java index 5888d18affc..7eff9f7a473 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java @@ -31,7 +31,6 @@ import com.fortify.cli.aviator.audit.model.FPRAuditResult; import com.fortify.cli.aviator.audit.model.FilterSelection; import com.fortify.cli.aviator.audit.model.ParsedFprData; -import com.fortify.cli.aviator.config.IAviatorLogger; import com.fortify.cli.aviator.config.TagMappingConfig; import com.fortify.cli.aviator.fpr.FPRProcessor; import com.fortify.cli.aviator.fpr.Vulnerability; @@ -72,10 +71,7 @@ public static FPRAuditResult auditFPR(AuditFprOptions options) // --- STAGE 3: AUDITING --- Map auditResponses = new ConcurrentHashMap<>(); - AuditOutcome auditOutcome = performAviatorAudit( - parsedData, options.getLogger(), options.getToken(), options.getAppVersion(), options.getUrl(), options.getSscAppName(), options.getSscAppVersion(), - auditResponses, filterSelection, options.getFprHandle(), options.getFolderPriorityOrder(), sourceDecoder - ); + AuditOutcome auditOutcome = performAviatorAudit(parsedData, auditResponses, filterSelection, options); // --- STAGE 4: FINALIZATION --- return finalizeFprAudit( @@ -127,32 +123,23 @@ private static Map buildIssueCategoryLookup(List return issueCategoryLookup; } - private static AuditOutcome performAviatorAudit( - ParsedFprData parsedData, IAviatorLogger logger, - String token, String appVersion, String url, String sscAppName, String sscAppVersion, - Map auditResponsesToFill, FilterSelection filterSelection, FprHandle fprHandle, - List folderPriorityOrder, ISourceDecoder sourceDecoder) { + private static AuditOutcome performAviatorAudit(ParsedFprData parsedData, Map auditResponsesToFill, + FilterSelection filterSelection, AuditFprOptions options) { SourceLanguageResolver sourceLanguageResolver = new SourceLanguageResolver(parsedData.streamingFVDLProcessor.getFvdlMetadata()); parsedData.streamingFVDLProcessor.getFvdlMetadata().clearSourceFileTypeIndexes(); - IssueAuditor issueAuditor = new IssueAuditor( - parsedData.vulnerabilities, - parsedData.auditProcessor, - parsedData.auditIssueMap, - parsedData.fprInfo, - sscAppName, - sscAppVersion, - filterSelection, - logger, - folderPriorityOrder, - sourceLanguageResolver, - sourceDecoder, - parsedData.streamingFVDLProcessor.getFvdlMetadata() - ); - return issueAuditor.performAudit( - auditResponsesToFill, token, appVersion, parsedData.fprInfo.getBuildId(), url, fprHandle - ); + IssueAuditor issueAuditor = IssueAuditor.builder() + .vulnerabilities(parsedData.vulnerabilities) + .auditProcessor(parsedData.auditProcessor) + .auditIssueMap(parsedData.auditIssueMap) + .fprInfo(parsedData.fprInfo) + .filterSelection(filterSelection) + .sourceLanguageResolver(sourceLanguageResolver) + .fvdlMetadata(parsedData.streamingFVDLProcessor.getFvdlMetadata()) + .options(options) + .build(); + return issueAuditor.performAudit(auditResponsesToFill); } private static FPRAuditResult finalizeFprAudit( diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java index dc5b6897892..40bb0d4c4e5 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java @@ -14,9 +14,11 @@ import static com.fortify.cli.aviator.util.Constants.DEFAULT_PING_INTERVAL_SECONDS; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -36,11 +38,12 @@ import com.fortify.cli.aviator._common.exception.AviatorQuotaFilterException; import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; +import com.fortify.cli.aviator.audit.model.AuditFprOptions; import com.fortify.cli.aviator.audit.model.AuditOutcome; import com.fortify.cli.aviator.audit.model.AuditResponse; import com.fortify.cli.aviator.audit.model.FilterSelection; import com.fortify.cli.aviator.audit.model.UserPrompt; -import com.fortify.cli.aviator.config.IAviatorLogger; +import com.fortify.cli.aviator.config.TagMappingConfig; import com.fortify.cli.aviator.fpr.Vulnerability; import com.fortify.cli.aviator.fpr.filter.Filter; import com.fortify.cli.aviator.fpr.filter.FilterSet; @@ -51,14 +54,14 @@ import com.fortify.cli.aviator.fpr.model.FPRInfo; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.fpr.processor.AuditProcessor; -import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; -import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.grpc.AviatorGrpcClient; import com.fortify.cli.aviator.grpc.AviatorGrpcClientHelper; import com.fortify.cli.aviator.util.Constants; -import com.fortify.cli.aviator.util.FprHandle; +import com.fortify.cli.aviator.util.ResourceUtil; import com.fortify.cli.aviator.util.StringUtil; +import lombok.Builder; + public class IssueAuditor { @@ -84,27 +87,17 @@ public class IssueAuditor { private TagDefinition humanAuditTag; private TagDefinition aviatorStatusTag; private final SourceLanguageResolver sourceLanguageResolver; - private final ISourceDecoder sourceDecoder; private final FVDLMetadata fvdlMetadata; + private final AuditFprOptions options; + private final Set resultTagIds; - private final IAviatorLogger logger; - private final List customPriorityOrder; - - public IssueAuditor(List vulnerabilities, AuditProcessor auditProcessor, Map auditIssueMap, - FPRInfo fprInfo, String SSCApplicationName, String SSCApplicationVersion, - FilterSelection filterSelection, IAviatorLogger logger, List customPriorityOrder, - SourceLanguageResolver sourceLanguageResolver) { - this(vulnerabilities, auditProcessor, auditIssueMap, fprInfo, SSCApplicationName, SSCApplicationVersion, - filterSelection, logger, customPriorityOrder, sourceLanguageResolver, SourceDecoders.defaults(), null); - } - + @Builder public IssueAuditor(List vulnerabilities, AuditProcessor auditProcessor, Map auditIssueMap, - FPRInfo fprInfo, String SSCApplicationName, String SSCApplicationVersion, - FilterSelection filterSelection, IAviatorLogger logger, List customPriorityOrder, - SourceLanguageResolver sourceLanguageResolver, ISourceDecoder sourceDecoder, - FVDLMetadata fvdlMetadata) { - this.logger = logger; - this.customPriorityOrder = customPriorityOrder; + FPRInfo fprInfo, FilterSelection filterSelection, SourceLanguageResolver sourceLanguageResolver, + FVDLMetadata fvdlMetadata, AuditFprOptions options) { + Objects.requireNonNull(options, "options"); + Objects.requireNonNull(options.getSourceDecoder(), "sourceDecoder"); + this.options = options; this.MAX_PER_CATEGORY = Constants.MAX_PER_CATEGORY; this.MAX_TOTAL = Constants.MAX_TOTAL; this.MAX_PER_CATEGORY_EXCEEDED = Constants.MAX_PER_CATEGORY_EXCEEDED; @@ -115,13 +108,13 @@ public IssueAuditor(List vulnerabilities, AuditProcessor auditPro this.auditIssueMap = auditIssueMap; this.fprInfo = fprInfo; this.filterSelection = filterSelection; - this.SSCApplicationName = SSCApplicationName; - this.SSCApplicationVersion = SSCApplicationVersion; + this.SSCApplicationName = options.getSscAppName(); + this.SSCApplicationVersion = options.getSscAppVersion(); this.sourceLanguageResolver = sourceLanguageResolver; - this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); this.fvdlMetadata = fvdlMetadata; this.analysisTag = fprInfo.getFilterTemplate().getTagDefinitions().stream().filter(t -> "Analysis".equalsIgnoreCase(t.getName())).findFirst().orElse(null); this.resultsTag = resolveResultTag("", "", analysisTag); + this.resultTagIds = resolveResultTagIds(); } private TagDefinition resolveResultTag(String tagName, String tagGuid, TagDefinition analysisTag) { @@ -165,30 +158,30 @@ private TagDefinition resolveHumanAuditStatus() { return new TagDefinition(name, id, values, false); } - public AuditOutcome performAudit(Map auditResponses, String token, - String projectName, String projectBuildId, String url, FprHandle fprHandle) { - projectName = StringUtil.isEmpty(projectName) ? projectBuildId : projectName; - logger.progress("Starting audit for project: %s", projectName); + public AuditOutcome performAudit(Map auditResponses) { + String projectName = StringUtil.isEmpty(options.getAppVersion()) ? fprInfo.getBuildId() : options.getAppVersion(); + options.getLogger().progress("Starting audit for project: %s", projectName); ConcurrentLinkedDeque promptsToAudit = prepareAndFilterPrompts(); int totalIssuesToAudit = promptsToAudit.size(); - logger.progress("Final count of issues to be audited: %d", totalIssuesToAudit); + options.getLogger().progress("Final count of issues to be audited: %d", totalIssuesToAudit); if (promptsToAudit.isEmpty()) { - logger.progress("Audit skipped - no issues to process after filtering."); + options.getLogger().progress("Audit skipped - no issues to process after filtering."); } else { - try (AviatorGrpcClient client = AviatorGrpcClientHelper.createClient(url, logger, DEFAULT_PING_INTERVAL_SECONDS)) { + try (AviatorGrpcClient client = AviatorGrpcClientHelper.createClient(options.getUrl(), options.getLogger(), DEFAULT_PING_INTERVAL_SECONDS)) { CompletableFuture> future = client.processBatchRequests(promptsToAudit, projectName, fprInfo.getBuildId(), SSCApplicationName, - SSCApplicationVersion, token, fprHandle, customPriorityOrder, sourceDecoder, fvdlMetadata); + SSCApplicationVersion, options.getToken(), options.getFprHandle(), + options.getFolderPriorityOrder(), options.getSourceDecoder(), fvdlMetadata); Map responses = future.get(500, TimeUnit.MINUTES); responses.forEach((requestId, response) -> auditResponses.put(response.getIssueId(), response)); - logger.progress("Audit completed"); + options.getLogger().progress("Audit completed"); } catch (ExecutionException e) { Throwable cause = e.getCause(); // Handle quota filtering exception (all issues filtered out) if (cause instanceof AviatorQuotaFilterException) { - logger.progress("All issues filtered out due to quota constraints: %s", cause.getMessage()); + options.getLogger().progress("All issues filtered out due to quota constraints: %s", cause.getMessage()); // Update totalIssuesToAudit to reflect actual auditable count (0) totalIssuesToAudit = 0; } else if (cause instanceof AviatorSimpleException) { @@ -199,10 +192,10 @@ public AuditOutcome performAudit(Map auditResponses, Stri throw new AviatorTechnicalException("Unexpected error during audit execution", cause); } } catch (TimeoutException e) { - logger.error("Audit failed due to timeout after 500 minutes"); + options.getLogger().error("Audit failed due to timeout after 500 minutes"); throw new AviatorTechnicalException("Audit timed out after 500 minutes", e); } catch (InterruptedException e) { - logger.error("Audit failed due to interruption"); + options.getLogger().error("Audit failed due to interruption"); Thread.currentThread().interrupt(); throw new AviatorTechnicalException("Audit interrupted", e); } @@ -243,31 +236,131 @@ private ConcurrentLinkedDeque prepareAndFilterPrompts() { private boolean shouldInclude(UserPrompt userPrompt) { + AuditIssue auditIssue = auditIssue(userPrompt); + if (options.isForceReaudit()) { + if (hasHumanTriage(auditIssue)) { + LOG.debug("Skipping force re-audit for suppressed or human-triaged issue ID: {}", + userPrompt.getIssueData().getInstanceID()); + return false; + } + if (isAviatorWork(auditIssue)) { + LOG.debug("Including previously processed Aviator issue for force re-audit: {}", + userPrompt.getIssueData().getInstanceID()); + return true; + } + } if (isAudited(userPrompt)) { LOG.debug("Skipping already audited issue ID: {}", userPrompt.getIssueData().getInstanceID()); return false; } - if (humanAuditTag != null) { - String issueId = userPrompt.getIssueData().getInstanceID(); - String status = Optional.ofNullable(auditIssueMap.get(issueId)).map(AuditIssue::getTags).map(tags -> tags.get("604f0fbe-b5fe-47cd-a9cb-587ad8ebe93a")).orElse(null); - if (!StringUtil.isEmpty(status) && !Constants.PENDING_REVIEW.equalsIgnoreCase(status)) { - LOG.debug("Skipping because already manually audited: {}", issueId); - return false; + if (humanAuditTag != null && isManuallyAudited(auditIssue)) { + LOG.debug("Skipping because already manually audited: {}", userPrompt.getIssueData().getInstanceID()); + return false; + } + + if (aviatorStatusTag != null && isProcessedByAviator(auditIssue)) { + LOG.debug("Skipping already processed by Aviator: {}", userPrompt.getIssueData().getInstanceID()); + return false; + } + + return true; + } + + private AuditIssue auditIssue(UserPrompt userPrompt) { + return auditIssueMap.get(userPrompt.getIssueData().getInstanceID()); + } + + private static boolean isManuallyAudited(AuditIssue auditIssue) { + if (auditIssue == null || auditIssue.getTags() == null) { + return false; + } + return !StringUtil.isPendingReviewValue(auditIssue.getTags().get(Constants.FOD_TAG_ID)); + } + + private boolean isProcessedByAviator(AuditIssue auditIssue) { + if (auditIssue == null || auditIssue.getTags() == null) { + return false; + } + String status = auditIssue.getTags().get(Constants.AVIATOR_STATUS_TAG_ID); + return !StringUtil.isEmpty(status) && Constants.PROCESSED_BY_AVIATOR.equalsIgnoreCase(status); + } + + private boolean hasHumanTriage(AuditIssue auditIssue) { + if (auditIssue == null) { + return false; + } + if (auditIssue.isSuppressed()) { + return true; + } + boolean processedByAviator = isProcessedByAviator(auditIssue); + for (String tagId : resultTagIds) { + if (!isResultTagSet(auditIssue, tagId)) { + continue; + } + String username = lastWriterUsername(auditIssue, tagId); + if (Constants.isAviatorAuditUsername(username)) { + continue; + } + if (!StringUtil.isEmpty(username) || Constants.FOD_TAG_ID.equalsIgnoreCase(tagId) || !processedByAviator) { + return true; } } + return false; + } - if (aviatorStatusTag != null) { - String issueId = userPrompt.getIssueData().getInstanceID(); - String status = Optional.ofNullable(auditIssueMap.get(issueId)).map(AuditIssue::getTags).map(tags -> tags.get("FB7B0462-2C2E-46D9-811A-DCC1F3C83051")).orElse(null); - if (!StringUtil.isEmpty(status) && Constants.PROCESSED_BY_AVIATOR.equalsIgnoreCase(status)) { - LOG.debug("Skipping already PROCESSED_BY_AVIATOR: {}", issueId); - return false; + private boolean isAviatorWork(AuditIssue auditIssue) { + if (isProcessedByAviator(auditIssue)) { + return true; + } + for (String tagId : resultTagIds) { + if (isResultTagSet(auditIssue, tagId) && Constants.isAviatorAuditUsername(lastWriterUsername(auditIssue, tagId))) { + return true; } } + return false; + } - return true; + private boolean isResultTagSet(AuditIssue auditIssue, String tagId) { + Map tags = auditIssue.getTags(); + return tags != null && !StringUtil.isPendingReviewValue(tags.get(tagId)); + } + + private String lastWriterUsername(AuditIssue auditIssue, String tagId) { + Map lastTagUsernames = auditIssue.getLastTagUsernames(); + if (lastTagUsernames == null) { + return null; + } + String username = lastTagUsernames.get(tagId); + return StringUtil.isEmpty(username) ? null : username; + } + + private Set resolveResultTagIds() { + Set tagIds = new LinkedHashSet<>(List.of( + Constants.ANALYSIS_TAG_ID, Constants.AUDITOR_STATUS_TAG_ID, Constants.FOD_TAG_ID)); + if (analysisTag != null && !StringUtil.isEmpty(analysisTag.getId())) { + tagIds.add(analysisTag.getId()); + } + String mappedTagId = mappedTagId(); + if (!StringUtil.isEmpty(mappedTagId)) { + tagIds.add(mappedTagId); + } + return tagIds; + } + + private String mappedTagId() { + String tagMappingPath = options.getTagMappingPath(); + if (tagMappingPath == null || tagMappingPath.isBlank()) { + return null; + } + try { + TagMappingConfig config = ResourceUtil.loadYamlFile(new File(tagMappingPath), TagMappingConfig.class); + return config == null ? null : config.getTag_id(); + } catch (Exception e) { + LOG.debug("Could not read tag mapping file {}", tagMappingPath, e); + return null; + } } private boolean isAudited(UserPrompt userPrompt) { @@ -282,7 +375,7 @@ private boolean isAudited(UserPrompt userPrompt) { String auditorStatusTag = Constants.AUDITOR_STATUS_TAG_ID; String auditorStatusValue = tags.get(auditorStatusTag); - if (auditorStatusValue != null && !auditorStatusValue.equalsIgnoreCase("Pending Review")) { + if (!StringUtil.isPendingReviewValue(auditorStatusValue)) { return true; } @@ -299,12 +392,12 @@ private boolean isAudited(UserPrompt userPrompt) { if (analysisTag != null && tags.containsKey(analysisTag.getId())) { String tagValue = tags.get(analysisTag.getId()); - if (tagValue != null && !tagValue.equalsIgnoreCase("Not Set") && !tagValue.equalsIgnoreCase(Constants.PENDING_REVIEW) && !tagValue.trim().isEmpty()) { + if (!StringUtil.isPendingReviewValue(tagValue)) { return true; } } - if (tags.containsKey(analysisTagS) && !tags.get(analysisTagS).equalsIgnoreCase("Not Set") && !StringUtil.isEmpty(tags.get(analysisTagS))) { + if (tags.containsKey(analysisTagS) && !StringUtil.isPendingReviewValue(tags.get(analysisTagS))) { return true; } } @@ -352,7 +445,7 @@ private List filterVulnerabilities(List allVulnera .flatMap(List::stream) .distinct() .collect(Collectors.toList()); - logger.info("FilterSet '{}' applied. {} of {} total vulnerabilities remain.", fs.getTitle(), result.size(), allVulnerabilities.size()); + options.getLogger().info("FilterSet '{}' applied. {} of {} total vulnerabilities remain.", fs.getTitle(), result.size(), allVulnerabilities.size()); return result; } else { Set targetFolderIds = fs.getFolderDefinitions().stream() @@ -371,7 +464,7 @@ private List filterVulnerabilities(List allVulnera .distinct() .collect(Collectors.toList()); - logger.info("Filtered by folder(s) '{}'. {} vulnerabilities remain.", targetFolderNames, result.size()); + options.getLogger().info("Filtered by folder(s) '{}'. {} vulnerabilities remain.", targetFolderNames, result.size()); return result; } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java index 8fa13198222..f05b9dffa59 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java @@ -36,5 +36,6 @@ public class AuditFprOptions { private final boolean noFilterSet; private final List folderNames; private final List folderPriorityOrder; + @Builder.Default private final boolean forceReaudit = false; @Builder.Default private final ISourceDecoder sourceDecoder = SourceDecoders.defaults(); } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/FPRProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/FPRProcessor.java index b0899d4f593..b8744a29020 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/FPRProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/FPRProcessor.java @@ -146,7 +146,7 @@ private boolean isAudited(AuditIssue auditIssue) { } String auditorStatusValue = tags.get(Constants.AUDITOR_STATUS_TAG_ID); - if (!isPendingReviewValue(auditorStatusValue)) { + if (!StringUtil.isPendingReviewValue(auditorStatusValue)) { return true; } @@ -159,16 +159,7 @@ private boolean isAudited(AuditIssue auditIssue) { } String analysisTagValue = tags.get(Constants.ANALYSIS_TAG_ID); - return analysisTagValue != null - && !analysisTagValue.equalsIgnoreCase("Not Set") - && !analysisTagValue.equalsIgnoreCase(Constants.PENDING_REVIEW) - && !StringUtil.isEmpty(analysisTagValue); - } - - private boolean isPendingReviewValue(String value) { - return StringUtil.isEmpty(value) - || value.equalsIgnoreCase("Pending Review") - || value.equalsIgnoreCase(Constants.PENDING_REVIEW); + return !StringUtil.isPendingReviewValue(analysisTagValue); } private String resolveIssueStatus(AuditIssue auditIssue) { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/AuditIssue.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/AuditIssue.java index 75c616be25c..f62638a885a 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/AuditIssue.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/AuditIssue.java @@ -30,6 +30,7 @@ public class AuditIssue { private boolean suppressed; private int revision; @Builder.Default private Map tags = new HashMap<>(); + @Builder.Default private Map lastTagUsernames = new HashMap<>(); @Builder.Default private List threadedComments = new ArrayList<>(); public void addTag(String tagId, String tagValue) { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java index d919ff321b6..11a79e38abf 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java @@ -249,6 +249,7 @@ private AuditIssue processAuditIssue(Element issueElement) { tags.put(tagId, tagValue); } auditIssueBuilder.tags(tags); + auditIssueBuilder.lastTagUsernames(lastTagUsernames(issueElement)); List threadedComments = new ArrayList<>(); NodeList commentNodes = issueElement.getElementsByTagNameNS(AUDIT_NAMESPACE_URI, "Comment"); @@ -267,6 +268,24 @@ private AuditIssue processAuditIssue(Element issueElement) { } + private Map lastTagUsernames(Element issueElement) { + Map lastTagUsernames = new HashMap<>(); + NodeList tagHistories = issueElement.getElementsByTagNameNS(AUDIT_NAMESPACE_URI, "TagHistory"); + for (int index = 0; index < tagHistories.getLength(); index++) { + Element tagHistory = (Element) tagHistories.item(index); + NodeList tags = tagHistory.getElementsByTagNameNS(AUDIT_NAMESPACE_URI, "Tag"); + if (tags.getLength() == 0) { + continue; + } + String tagId = ((Element) tags.item(0)).getAttribute("id"); + String username = Optional.ofNullable(getFirstElementContentNS(tagHistory, "Username")).orElse(""); + if (tagId != null && !tagId.isBlank()) { + lastTagUsernames.put(tagId, username); + } + } + return lastTagUsernames; + } + private String getTagValue(Element tagElement) { NodeList valueNodes = tagElement.getElementsByTagNameNS(AUDIT_NAMESPACE_URI, "Value"); if (valueNodes.getLength() > 0) { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java index 2a059b8380b..2f717dd66b8 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import java.util.zip.ZipFile; import javax.xml.parsers.DocumentBuilder; @@ -85,6 +86,7 @@ private enum SkipReason { REMEDIATION_DATA_INVALID("Remediation data invalid"), REMEDIATION_LINE_RANGE_INVALID("Remediation line range invalid"), SOURCE_CONTEXT_NOT_FOUND("Source context not found"), + SOURCE_CONTEXT_AMBIGUOUS("Source context matched multiple locations"), ORIGINAL_CODE_NOT_FOUND("Original code not found"), REMEDIATION_ENCODE_FAILED("Remediation encode failed"), SOURCE_WRITE_FAILED("Source file write failed"), @@ -211,7 +213,7 @@ private boolean processRemediation(Element remediation, Path sourceBasePath, FVD try { Map pendingWrites = prepareFileChanges(remediation, sourceBasePath, fvdlMetadata); if (pendingWrites.isEmpty()) { - recordSkipped(skippedByReason, SkipReason.NO_CHANGES); + recordSkipped(skippedByReason, SkipReason.NO_CHANGES.displayName); return false; } try { @@ -222,15 +224,15 @@ private boolean processRemediation(Element remediation, Path sourceBasePath, FVD throw new SkipRemediationException(SkipReason.SOURCE_WRITE_FAILED, e.getMessage(), e); } } catch (SkipRemediationException e) { - recordSkipped(skippedByReason, e.reason); - LOG.info("Skipping remediation {}: {}", instanceId, e.getMessage()); + recordSkipped(skippedByReason, skipReasonLabel(e)); + LOG.warn("Skipping remediation {}: {}", instanceId, e.getMessage()); LOG.debug("Skip reason for remediation {}: {}", instanceId, e.reason.displayName, e); return false; } catch (RollbackRemediationException e) { throw e; } catch (Exception e) { - recordSkipped(skippedByReason, SkipReason.UNEXPECTED_ERROR); - LOG.info("Skipping remediation {} due to an unexpected processing error", instanceId); + recordSkipped(skippedByReason, SkipReason.UNEXPECTED_ERROR.displayName); + LOG.warn("Skipping remediation {} due to an unexpected processing error", instanceId); LOG.debug("Unexpected error while processing remediation {}", instanceId, e); return false; } @@ -307,7 +309,8 @@ private String applyChange(String instanceId, String filename, String fileHash, LOG.debug("Remediation {} hash check for '{}': {}", instanceId, filename, fileHashMatches ? "matched" : "mismatched"); if (!fileHashMatches) { LOG.debug("File hash mismatch for remediation {} in {}; searching changed source content", instanceId, filename); - String contextText = getRequiredElementText(change, "Context"); + Element contextElement = getRequiredElement(change, "Context"); + String contextText = contextElement.getTextContent(); List contextLine = Arrays.asList(contextText.split("\\r?\\n")); int contextLineFrom = fuzzySearchContext(instanceId, filename, originalLines, contextLine); if (contextLineFrom == -1) { @@ -320,7 +323,10 @@ private String applyChange(String instanceId, String filename, String fileHash, String originalCodeText = getRequiredElementText(change, "OriginalCode"); List originalCodeLine = Arrays.asList(originalCodeText.split("\\r?\\n")); - int[] lineFromTo = fuzzySearchOriginalCode(instanceId, filename, originalLines, originalCodeLine, contextLineFrom); + int contextBefore = parseRequiredContextAttribute(contextElement, "before"); + int contextAfter = parseRequiredContextAttribute(contextElement, "after"); + int[] lineFromTo = fuzzySearchOriginalCode(instanceId, filename, originalLines, originalCodeLine, + contextLineFrom, contextLine.size(), contextBefore, contextAfter); if (lineFromTo[0] == -1 || lineFromTo[1] == -1) { LOG.debug("Original code search failed for remediation {} in {}; context line={}, original code lines={}, source lines={}", instanceId, filename, contextLineFrom + 1, originalCodeLine.size(), originalLines.size()); @@ -387,7 +393,15 @@ private void rollbackRemediationWrites(String instanceId, List originalLines, List contextLine) { try { - return FuzzyContextSearcher.fuzzySearchContext(originalLines, contextLine, 0); + List matches = FuzzyContextSearcher.fuzzySearchContextMatches(originalLines, contextLine, 0); + if (matches.size() > 1) { + String candidateLines = matches.stream() + .map(line -> String.valueOf(line + 1)) + .collect(Collectors.joining(", ")); + throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_AMBIGUOUS, + "Source context matched multiple locations in file '" + filename + "'; candidate lines: " + candidateLines); + } + return matches.isEmpty() ? -1 : matches.get(0); } catch (IOException e) { throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_NOT_FOUND, "Error searching source context for remediation '" + instanceId + "' in file '" + filename + "'", e); @@ -395,8 +409,19 @@ private int fuzzySearchContext(String instanceId, String filename, List } private int[] fuzzySearchOriginalCode(String instanceId, String filename, List originalLines, List originalCodeLine, - int contextLineFrom) { - return FuzzyContextSearcher.fuzzySearchOriginalCode(originalLines, originalCodeLine, 0, contextLineFrom); + int contextLineFrom, int contextLineCount, int contextBefore, int contextAfter) { + int contextStart = contextLineFrom + contextBefore; + int contextEnd = contextLineFrom + contextLineCount - contextAfter; + if (contextStart < 0 || contextStart >= contextEnd || contextEnd > originalLines.size()) { + return new int[] {-1, -1}; + } + + int[] lineFromTo = FuzzyContextSearcher.fuzzySearchOriginalCode( + originalLines.subList(contextStart, contextEnd), originalCodeLine, 0, 0); + if (lineFromTo[0] == -1 || lineFromTo[1] == -1) { + return lineFromTo; + } + return new int[] {lineFromTo[0] + contextStart, lineFromTo[1] + contextStart}; } private boolean isFilePresent(Path path) { @@ -447,12 +472,34 @@ private byte[] encodeSourceFile(String content, Charset charset, String filename } private String getRequiredElementText(Element parent, String elementName) { + return getRequiredElement(parent, elementName).getTextContent(); + } + + private Element getRequiredElement(Element parent, String elementName) { NodeList nodes = parent.getElementsByTagNameNS(NAMESPACE_URI, elementName); if (nodes.getLength() == 0 || nodes.item(0) == null) { throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, "Missing required remediation element '" + elementName + "'"); } - return nodes.item(0).getTextContent(); + return (Element) nodes.item(0); + } + + private int parseRequiredContextAttribute(Element context, String attributeName) { + String value = context.getAttribute(attributeName); + if (value == null || value.isBlank()) { + throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, + "Missing required remediation context attribute '" + attributeName + "'"); + } + try { + int parsedValue = Integer.parseInt(value); + if (parsedValue < 0) { + throw new NumberFormatException("negative value"); + } + return parsedValue; + } catch (NumberFormatException e) { + throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, + "Invalid remediation context attribute '" + attributeName + "': " + value, e); + } } private int parseRequiredInt(Element parent, String elementName) { @@ -517,8 +564,12 @@ private String calculateHashBase64(String content, String algorithm) { } } - private void recordSkipped(Map skippedByReason, SkipReason reason) { - skippedByReason.merge(reason.displayName, 1, Integer::sum); + private void recordSkipped(Map skippedByReason, String reason) { + skippedByReason.merge(reason, 1, Integer::sum); + } + + private String skipReasonLabel(SkipRemediationException exception) { + return exception.reason.displayName; } private String formatSkippedReasons(Map skippedByReason) { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/Constants.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/Constants.java index 756789a0883..00613cd242b 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/Constants.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/Constants.java @@ -12,6 +12,9 @@ */ package com.fortify.cli.aviator.util; +import java.util.Locale; +import java.util.Set; + public class Constants { // Audit Result Values @@ -45,8 +48,19 @@ public class Constants { public static final String AUDITOR_STATUS_TAG_ID = "ACB05E55-E74D-468C-8501-52E1FDC27D71"; public static final String FOD_TAG_ID = "604f0fbe-b5fe-47cd-a9cb-587ad8ebe93a"; - // User Names + // User Names written into audit.xml TagHistory / comments public static final String USER_NAME = "Fortify Remediation Aviator"; + public static final String USER_NAME_LEGACY_FORTIFY_AVIATOR = "Fortify Aviator"; + public static final String USER_NAME_LEGACY_CORE_SAST_AVIATOR = "Core SAST Aviator"; + private static final Set AVIATOR_AUDIT_USERNAMES = Set.of( + USER_NAME.toLowerCase(Locale.ROOT), + USER_NAME_LEGACY_FORTIFY_AVIATOR.toLowerCase(Locale.ROOT), + USER_NAME_LEGACY_CORE_SAST_AVIATOR.toLowerCase(Locale.ROOT) + ); + + public static boolean isAviatorAuditUsername(String username) { + return username != null && AVIATOR_AUDIT_USERNAMES.contains(username.trim().toLowerCase(Locale.ROOT)); + } // Other Constants public static final String AUDIT_NAMESPACE_URI = "xmlns://www.fortify.com/schema/audit"; diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FuzzyContextSearcher.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FuzzyContextSearcher.java index 5b3ff717364..4a4b8e6dd51 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FuzzyContextSearcher.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FuzzyContextSearcher.java @@ -28,58 +28,68 @@ public class FuzzyContextSearcher { */ public static int fuzzySearchContext(List sourceLines, List contextLines, int maxMismatches) throws IOException { + List matches = fuzzySearchContextMatches(sourceLines, contextLines, maxMismatches); + return matches.isEmpty() ? -1 : matches.get(0); + } + + public static List fuzzySearchContextMatches(List sourceLines, List contextLines, + int maxMismatches) throws IOException { List normalizedSource = normalizeLines(sourceLines); List normalizedContext = normalizeLines(contextLines); + List matches = new ArrayList<>(); + boolean contextStartsWithBlank = !normalizedContext.isEmpty() && normalizedContext.get(0).isEmpty(); for (int i = 0; i < normalizedSource.size(); i++) { - int mismatchCount = 0; - int sourceIndex = i; - int contextIndex = 0; - boolean similar = false; + if (isUnusableContextStart(normalizedSource, i, contextStartsWithBlank)) { + continue; + } + Integer matchStart = findContextMatchStart(normalizedSource, normalizedContext, maxMismatches, i); + if (matchStart != null && !matches.contains(matchStart)) { + matches.add(matchStart); + } + } - while (contextIndex < normalizedContext.size() && sourceIndex < normalizedSource.size()) { - String contextLine = normalizedContext.get(contextIndex).trim(); + return List.copyOf(matches); + } - if (contextLine.isEmpty()) { - contextIndex++; // Skip empty context lines - continue; - } + private static boolean isUnusableContextStart(List normalizedSource, int index, boolean contextStartsWithBlank) { + boolean sourceStartsWithBlank = normalizedSource.get(index).isEmpty(); + if (!contextStartsWithBlank) { + return sourceStartsWithBlank; + } + return !sourceStartsWithBlank || (index > 0 && normalizedSource.get(index - 1).isEmpty()); + } - // Skip empty source lines too - String sourceLine = normalizedSource.get(sourceIndex).trim(); - while (sourceLine.isEmpty()) { - sourceIndex++; - if (sourceIndex >= normalizedSource.size()) { - break; - } - sourceLine = normalizedSource.get(sourceIndex).trim(); - if(!similar) - i = sourceIndex; - } + private static Integer findContextMatchStart(List normalizedSource, List normalizedContext, + int maxMismatches, int startIndex) { + int mismatchCount = 0; + int sourceIndex = startIndex; + int contextIndex = 0; - if (sourceIndex >= normalizedSource.size()) { - break; // No more source lines to match - } + while (contextIndex < normalizedContext.size() && sourceIndex < normalizedSource.size()) { + String contextLine = normalizedContext.get(contextIndex); + if (contextLine.isEmpty()) { + contextIndex++; + continue; + } - similar = linesSimilar(sourceLine, contextLine); + sourceIndex = skipEmptySourceLines(normalizedSource, sourceIndex); + if (sourceIndex >= normalizedSource.size()) { + break; + } - if (!similar) { - mismatchCount++; - if (mismatchCount > maxMismatches) { - break; - } + if (!linesSimilar(normalizedSource.get(sourceIndex), contextLine)) { + mismatchCount++; + if (mismatchCount > maxMismatches) { + break; } - - sourceIndex++; - contextIndex++; } - if (contextIndex == normalizedContext.size() && mismatchCount <= maxMismatches) { - return i; // Found approximate match starting at i (ignoring blanks) - } + sourceIndex++; + contextIndex++; } - return -1; // Not found + return contextIndex == normalizedContext.size() && mismatchCount <= maxMismatches ? startIndex : null; } public static int[] fuzzySearchOriginalCode(List sourceLines, List originalCodeLine, int maxMismatches, int startIndex) { @@ -143,7 +153,7 @@ private static int skipEmptySourceLines(List normalizedSource, int sourc private static List normalizeLines(List lines) { List result = new ArrayList<>(); for (String line : lines) { - result.add(line.trim().replaceAll("\\s+", " ")); + result.add(line.trim().replaceAll("\\s+", " ")); } return result; } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/StringUtil.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/StringUtil.java index adf83ec46e9..e92f131f91c 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/StringUtil.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/StringUtil.java @@ -23,6 +23,17 @@ public static boolean isEmpty(String test) { return test == null || test.length() == 0; } + public static boolean isPendingReviewValue(String value) { + if (value == null) { + return true; + } + String normalizedValue = value.trim(); + return normalizedValue.isEmpty() + || "Pending Review".equalsIgnoreCase(normalizedValue) + || "Not Set".equalsIgnoreCase(normalizedValue) + || Constants.PENDING_REVIEW.equalsIgnoreCase(normalizedValue); + } + /** * Strips HTML-like tags from a string to clean it up for display. * diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/IssueAuditorTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/IssueAuditorTest.java index 023d2d5bc5c..145dc6bddfd 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/IssueAuditorTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/IssueAuditorTest.java @@ -25,6 +25,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedDeque; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -33,19 +35,31 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import com.fortify.cli.aviator.audit.model.AuditFprOptions; import com.fortify.cli.aviator.audit.model.FilterSelection; +import com.fortify.cli.aviator.audit.model.UserPrompt; import com.fortify.cli.aviator.config.IAviatorLogger; import com.fortify.cli.aviator.fpr.Vulnerability; import com.fortify.cli.aviator.fpr.filter.Filter; import com.fortify.cli.aviator.fpr.filter.FilterSet; import com.fortify.cli.aviator.fpr.filter.FilterTemplate; +import com.fortify.cli.aviator.fpr.model.AuditIssue; import com.fortify.cli.aviator.fpr.model.FPRInfo; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; class IssueAuditorTest { + private static final String TEST_ISSUE_ID = "PROCESSED_ISSUE"; + private static final IAviatorLogger NO_OP_LOGGER = new IAviatorLogger() { + @Override public void progress(String format, Object... args) {} + @Override public void info(String format, Object... args) {} + @Override public void warn(String format, Object... args) {} + @Override public void error(String format, Object... args) {} + }; + private Path tempFprFile; private FprHandle fprHandle; @@ -113,6 +127,202 @@ void testFprInfoMissingBuildIdDefaultsToEmptyString() throws Exception { assertEquals("", fprInfo.getBuildId()); } + @Test + void skipsPreviouslyProcessedAviatorIssueWithoutForceReaudit() throws Exception { + IssueAuditor auditor = createIssueAuditor(false, false, + Map.of(Constants.AVIATOR_STATUS_TAG_ID, Constants.PROCESSED_BY_AVIATOR)); + + assertTrue(prepareIssueIds(auditor).isEmpty()); + } + + @Test + void forceReauditIncludesProcessedAviatorIssueAndIgnoresAviatorOutcomeTag() throws Exception { + IssueAuditor auditor = createIssueAuditor(true, false, Map.of( + Constants.AVIATOR_STATUS_TAG_ID, Constants.PROCESSED_BY_AVIATOR, + Constants.AVIATOR_EXPECTED_OUTCOME_TAG_ID, Constants.EXPLOITABLE)); + + assertEquals(List.of(TEST_ISSUE_ID), prepareIssueIds(auditor)); + } + + @Test + void forceReauditIncludesProcessedAviatorIssueWithAnalysisTag() throws Exception { + IssueAuditor auditor = createIssueAuditor(true, false, Map.of( + Constants.AVIATOR_STATUS_TAG_ID, Constants.PROCESSED_BY_AVIATOR, + Constants.ANALYSIS_TAG_ID, Constants.EXPLOITABLE)); + + assertEquals(List.of(TEST_ISSUE_ID), prepareIssueIds(auditor)); + } + + @Test + void forceReauditStillSkipsSuppressedIssue() throws Exception { + IssueAuditor auditor = createIssueAuditor(true, true, + Map.of(Constants.AVIATOR_STATUS_TAG_ID, Constants.PROCESSED_BY_AVIATOR)); + + assertTrue(prepareIssueIds(auditor).isEmpty()); + } + + @Test + void forceReauditStillSkipsHumanTriagedIssue() throws Exception { + IssueAuditor auditor = createIssueAuditor(true, false, Map.of( + Constants.AVIATOR_STATUS_TAG_ID, Constants.PROCESSED_BY_AVIATOR, + Constants.FOD_TAG_ID, Constants.EXPLOITABLE)); + + assertTrue(prepareIssueIds(auditor).isEmpty()); + } + + @Test + void forceReauditIncludesIssueWithPendingReviewState() throws Exception { + IssueAuditor auditor = createIssueAuditor(true, false, Map.of( + Constants.AVIATOR_STATUS_TAG_ID, Constants.PROCESSED_BY_AVIATOR, + Constants.FOD_TAG_ID, "Pending Review", + Constants.AUDITOR_STATUS_TAG_ID, Constants.PENDING_REVIEW)); + + assertEquals(List.of(TEST_ISSUE_ID), prepareIssueIds(auditor)); + } + + @Test + void treatsAllPendingAnalysisValuesAsUnaudited() throws Exception { + for (String pendingValue : List.of("Pending Review", "Not Set", "Pending Review/Not Set", " pending review ")) { + IssueAuditor auditor = createIssueAuditor(false, false, Map.of(Constants.ANALYSIS_TAG_ID, pendingValue)); + + assertEquals(List.of(TEST_ISSUE_ID), prepareIssueIds(auditor), pendingValue); + } + } + + @Test + void forceReauditStillSkipsIssueWithManualAuditorStatus() throws Exception { + IssueAuditor auditor = createIssueAuditor(true, false, Map.of( + Constants.AVIATOR_STATUS_TAG_ID, Constants.PROCESSED_BY_AVIATOR, + Constants.AUDITOR_STATUS_TAG_ID, Constants.EXPLOITABLE), + Map.of(Constants.AUDITOR_STATUS_TAG_ID, "analyst.user")); + + assertTrue(prepareIssueIds(auditor).isEmpty()); + } + + @Test + void forceReauditIncludesAviatorWrittenAuditorStatus() throws Exception { + IssueAuditor auditor = createIssueAuditor(true, false, Map.of( + Constants.AVIATOR_STATUS_TAG_ID, Constants.PROCESSED_BY_AVIATOR, + Constants.AUDITOR_STATUS_TAG_ID, Constants.EXPLOITABLE), + Map.of(Constants.AUDITOR_STATUS_TAG_ID, Constants.USER_NAME)); + + assertEquals(List.of(TEST_ISSUE_ID), prepareIssueIds(auditor)); + } + + @Test + void forceReauditSkipsHumanAnalysisOverrideAfterAviator() throws Exception { + IssueAuditor auditor = createIssueAuditor(true, false, Map.of( + Constants.AVIATOR_STATUS_TAG_ID, Constants.PROCESSED_BY_AVIATOR, + Constants.ANALYSIS_TAG_ID, Constants.EXPLOITABLE), + Map.of(Constants.ANALYSIS_TAG_ID, "analyst.user")); + + assertTrue(prepareIssueIds(auditor).isEmpty()); + } + + @Test + void forceReauditIncludesLegacyFortifyAviatorUsername() throws Exception { + IssueAuditor auditor = createIssueAuditor(true, false, Map.of( + Constants.AVIATOR_STATUS_TAG_ID, Constants.PROCESSED_BY_AVIATOR, + Constants.ANALYSIS_TAG_ID, Constants.EXPLOITABLE), + Map.of(Constants.ANALYSIS_TAG_ID, Constants.USER_NAME_LEGACY_FORTIFY_AVIATOR)); + + assertEquals(List.of(TEST_ISSUE_ID), prepareIssueIds(auditor)); + } + + @Test + void forceReauditIncludesMappedTagWrittenByAviatorWithoutStatusTag() throws Exception { + Path mappingFile = writeMappedTagFile(); + try { + IssueAuditor auditor = createIssueAuditor(true, false, + Map.of("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Constants.EXPLOITABLE), + Map.of("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Constants.USER_NAME), + mappingFile.toString()); + assertEquals(List.of(TEST_ISSUE_ID), prepareIssueIds(auditor)); + } finally { + Files.deleteIfExists(mappingFile); + } + } + + @Test + void forceReauditSkipsHumanWriterOnMappedTag() throws Exception { + Path mappingFile = writeMappedTagFile(); + try { + IssueAuditor auditor = createIssueAuditor(true, false, + Map.of("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Constants.EXPLOITABLE), + Map.of("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", "analyst.user"), + mappingFile.toString()); + assertTrue(prepareIssueIds(auditor).isEmpty()); + } finally { + Files.deleteIfExists(mappingFile); + } + } + + private Path writeMappedTagFile() throws IOException { + Path mappingFile = Files.createTempFile("tag-mapping", ".yaml"); + Files.writeString(mappingFile, "tag_id: \"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee\"\n"); + return mappingFile; + } + + private IssueAuditor createIssueAuditor(boolean forceReaudit, boolean suppressed, Map tags) { + return createIssueAuditor(forceReaudit, suppressed, tags, Map.of(), null); + } + + private IssueAuditor createIssueAuditor(boolean forceReaudit, boolean suppressed, Map tags, + Map lastTagUsernames) { + return createIssueAuditor(forceReaudit, suppressed, tags, lastTagUsernames, null); + } + + private IssueAuditor createIssueAuditor(boolean forceReaudit, boolean suppressed, Map tags, + Map lastTagUsernames, String tagMappingPath) { + FPRInfo fprInfo = new FPRInfo(fprHandle); + FilterTemplate filterTemplate = new FilterTemplate(); + filterTemplate.setTagDefinitions(new ArrayList<>()); + fprInfo.setFilterTemplate(filterTemplate); + + Vulnerability vulnerability = new Vulnerability(); + vulnerability.setInstanceID(TEST_ISSUE_ID); + AuditIssue auditIssue = AuditIssue.builder() + .instanceId(TEST_ISSUE_ID) + .suppressed(suppressed) + .tags(new HashMap<>(tags)) + .lastTagUsernames(new HashMap<>(lastTagUsernames)) + .build(); + + return IssueAuditor.builder() + .vulnerabilities(List.of(vulnerability)) + .auditIssueMap(Map.of(TEST_ISSUE_ID, auditIssue)) + .fprInfo(fprInfo) + .filterSelection(new FilterSelection(null, null)) + .sourceLanguageResolver(new SourceLanguageResolver(new FVDLMetadata())) + .options(auditOptions(forceReaudit, NO_OP_LOGGER, tagMappingPath)) + .build(); + } + + private AuditFprOptions auditOptions(boolean forceReaudit, IAviatorLogger logger) { + return auditOptions(forceReaudit, logger, null); + } + + private AuditFprOptions auditOptions(boolean forceReaudit, IAviatorLogger logger, String tagMappingPath) { + return AuditFprOptions.builder() + .fprHandle(fprHandle) + .logger(logger) + .sscAppName("TestApp") + .sscAppVersion("1.0") + .tagMappingPath(tagMappingPath) + .forceReaudit(forceReaudit) + .build(); + } + + @SuppressWarnings("unchecked") + private List prepareIssueIds(IssueAuditor auditor) throws Exception { + Method prepareMethod = IssueAuditor.class.getDeclaredMethod("prepareAndFilterPrompts"); + prepareMethod.setAccessible(true); + ConcurrentLinkedDeque prompts = (ConcurrentLinkedDeque) prepareMethod.invoke(auditor); + return prompts.stream() + .map(prompt -> prompt.getIssueData().getInstanceID()) + .collect(Collectors.toList()); + } + @Test void testFilterVulnerabilities_LegacySyntaxWithSpaces() throws Exception { @@ -159,11 +369,14 @@ void testFilterVulnerabilities_LegacySyntaxWithSpaces() throws Exception { List inputList = Arrays.asList(targetVuln, hiddenVuln); - IssueAuditor auditor = new IssueAuditor( - inputList, null, new HashMap<>(), fprInfo, - "TestApp", "1.0", selection, dummyLogger, null, - new SourceLanguageResolver(new FVDLMetadata()) - ); + IssueAuditor auditor = IssueAuditor.builder() + .vulnerabilities(inputList) + .auditIssueMap(new HashMap<>()) + .fprInfo(fprInfo) + .filterSelection(selection) + .sourceLanguageResolver(new SourceLanguageResolver(new FVDLMetadata())) + .options(auditOptions(false, dummyLogger)) + .build(); Method filterMethod = IssueAuditor.class.getDeclaredMethod("filterVulnerabilities", List.class, FilterSet.class); filterMethod.setAccessible(true); @@ -172,8 +385,8 @@ void testFilterVulnerabilities_LegacySyntaxWithSpaces() throws Exception { List results = (List) filterMethod.invoke(auditor, inputList, filterSet); List remainingIds = results.stream() - .map(Vulnerability::getInstanceID) - .collect(Collectors.toList()); + .map(Vulnerability::getInstanceID) + .collect(Collectors.toList()); assertEquals(1, remainingIds.size(), "Should verify that exactly one issue remains"); assertTrue(remainingIds.contains("TARGET_ISSUE"), diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/FPRProcessorTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/FPRProcessorTest.java index 7b7a94f007d..61b5734f54d 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/FPRProcessorTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/FPRProcessorTest.java @@ -128,6 +128,20 @@ void testProcessDoesNotMarkPendingReviewDefaultAuditorStatusAsAudited() throws E assertFalse(vulnerability.isAudited()); } + @Test + void testProcessTreatsAllPendingAnalysisValuesAsUnaudited() throws Exception { + createTestFpr(minimalAuditFvdl()); + + for (String pendingValue : List.of("Pending Review", "Not Set", "Pending Review/Not Set", " pending review ")) { + AuditIssue auditIssue = AuditIssue.builder() + .instanceId("instance-1") + .tags(Map.of(Constants.ANALYSIS_TAG_ID, pendingValue)) + .build(); + + assertFalse(processSingleVulnerability(auditIssue).isAudited(), pendingValue); + } + } + private String minimalAuditFvdl() { return """ diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/processor/AuditProcessorLastTagUsernamesTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/processor/AuditProcessorLastTagUsernamesTest.java new file mode 100644 index 00000000000..53a830524cd --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/processor/AuditProcessorLastTagUsernamesTest.java @@ -0,0 +1,129 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.processor; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.fortify.cli.aviator.fpr.model.AuditIssue; +import com.fortify.cli.aviator.util.Constants; +import com.fortify.cli.aviator.util.FprHandle; + +class AuditProcessorLastTagUsernamesTest { + private static final String INSTANCE_ID = "ISSUE-1"; + + @TempDir + Path tempDir; + + private FprHandle fprHandle; + + @AfterEach + void tearDown() throws Exception { + if (fprHandle != null) { + fprHandle.close(); + } + } + + @Test + void lastDocumentOrderTagHistoryWinsPerTag() throws Exception { + fprHandle = new FprHandle(createTestFpr(""" + + + + + + Exploitable + + + Exploitable + + + + Not an Issue + + %s + + + + Exploitable + + analyst.user + + + + Exploitable + + %s + + + + + """.formatted( + Constants.ANALYSIS_TAG_ID, Constants.AUDITOR_STATUS_TAG_ID, + Constants.ANALYSIS_TAG_ID, Constants.USER_NAME, + Constants.ANALYSIS_TAG_ID, + Constants.AUDITOR_STATUS_TAG_ID, Constants.USER_NAME_LEGACY_FORTIFY_AVIATOR))); + + Map issues = new AuditProcessor(fprHandle).processAuditXML(); + Map lastTagUsernames = issues.get(INSTANCE_ID).getLastTagUsernames(); + + assertEquals("analyst.user", lastTagUsernames.get(Constants.ANALYSIS_TAG_ID)); + assertEquals(Constants.USER_NAME_LEGACY_FORTIFY_AVIATOR, lastTagUsernames.get(Constants.AUDITOR_STATUS_TAG_ID)); + } + + @Test + void missingUsernameIsStoredAsEmptyString() throws Exception { + fprHandle = new FprHandle(createTestFpr(""" + + + + + + + Exploitable + + + + + + """.formatted(Constants.ANALYSIS_TAG_ID))); + + Map issues = new AuditProcessor(fprHandle).processAuditXML(); + + assertEquals("", issues.get(INSTANCE_ID).getLastTagUsernames().get(Constants.ANALYSIS_TAG_ID)); + } + + private Path createTestFpr(String auditXml) throws Exception { + Path fprPath = Files.createTempFile(tempDir, "audit-processor", ".fpr"); + try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(fprPath))) { + zipOutputStream.putNextEntry(new ZipEntry("audit.xml")); + zipOutputStream.write(auditXml.getBytes(StandardCharsets.UTF_8)); + zipOutputStream.closeEntry(); + + zipOutputStream.putNextEntry(new ZipEntry("src-archive/index.xml")); + zipOutputStream.write("".getBytes(StandardCharsets.UTF_8)); + zipOutputStream.closeEntry(); + } + return fprPath; + } +} diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessorTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessorTest.java new file mode 100644 index 00000000000..fe171bb4b71 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessorTest.java @@ -0,0 +1,145 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.processor; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.fortify.cli.aviator.util.FprHandle; + +class RemediationProcessorTest { + private static final String REMEDIATIONS_NAMESPACE = "xmlns://www.fortify.com/schema/remediations"; + + @TempDir + Path tempDir; + + @Test + void skipsAmbiguousContextWithoutChangingSource() throws Exception { + String originalSource = "before\nTARGET\nafter\nbefore\nTARGET\nafter\n"; + Path sourceFile = writeSourceFile(originalSource); + Path fprPath = createRemediationFpr(2, 2, 1, 1, "before\ntarget\nafter", "TARGET", "REPLACED"); + + RemediationProcessor.RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(1, metric.totalRemediations()); + assertEquals(0, metric.appliedRemediations()); + assertEquals(1, metric.skippedRemediations()); + assertEquals(Map.of( + "Source context matched multiple locations", 1), + metric.skippedByReason()); + assertEquals(originalSource, Files.readString(sourceFile)); + } + + @Test + void appliesRemediationWhenContextMatchesOnce() throws Exception { + Path sourceFile = writeSourceFile("before\nTARGET\nafter\n"); + Path fprPath = createRemediationFpr(2, 2, 1, 1, "before\ntarget\nafter", "TARGET", "REPLACED"); + + RemediationProcessor.RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(1, metric.appliedRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals("before\nREPLACED\nafter\n", Files.readString(sourceFile)); + } + + @Test + void appliesOriginalCodeAfterLeadingContextLines() throws Exception { + Path sourceFile = writeSourceFile("TARGET\nkeep\nTARGET\nafter\n"); + Path fprPath = createRemediationFpr(3, 3, 2, 1, "TARGET\nkeep\nTARGET\nafter", "TARGET", "REPLACED"); + + RemediationProcessor.RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(1, metric.appliedRemediations()); + assertEquals("TARGET\nkeep\nREPLACED\nafter\n", Files.readString(sourceFile)); + } + + @Test + void appliesRemediationWhenContextStartsWithBlankLine() throws Exception { + Path sourceFile = writeSourceFile("header\n\nTARGET\nafter\n"); + Path fprPath = createRemediationFpr(3, 3, 1, 1, "\nTARGET\nafter", "TARGET", "REPLACED"); + + RemediationProcessor.RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(1, metric.appliedRemediations()); + assertEquals("header\n\nREPLACED\nafter\n", Files.readString(sourceFile)); + } + + private Path writeSourceFile(String content) throws Exception { + Path sourceFile = tempDir.resolve("Example.java"); + Files.writeString(sourceFile, content, StandardCharsets.UTF_8); + return sourceFile; + } + + private Path createRemediationFpr(String context, String originalCode, String newCode) throws Exception { + return createRemediationFpr(2, 2, 1, 1, context, originalCode, newCode); + } + + private Path createRemediationFpr(int lineFrom, int lineTo, int contextBefore, int contextAfter, + String context, String originalCode, String newCode) throws Exception { + Path fprPath = tempDir.resolve("remediation.fpr"); + String remediationXml = """ + + + + test + 2026-08-26T00:00:00Z + + + + test + + Example.java + not-the-source-hash + + %d + %d + %s + %s + %s + + + + + + """.formatted(REMEDIATIONS_NAMESPACE, lineFrom, lineTo, contextBefore, contextAfter, + context, originalCode, newCode); + + try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(fprPath))) { + zipOutputStream.putNextEntry(new ZipEntry("remediations.xml")); + zipOutputStream.write(remediationXml.getBytes(StandardCharsets.UTF_8)); + zipOutputStream.closeEntry(); + } + return fprPath; + } +} diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/util/FuzzyContextSearcherTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/util/FuzzyContextSearcherTest.java index 22d662833b0..5e8cc165ebe 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/util/FuzzyContextSearcherTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/util/FuzzyContextSearcherTest.java @@ -13,6 +13,7 @@ package com.fortify.cli.aviator.util; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.List; @@ -41,4 +42,54 @@ void shouldMatchOriginalCodeAcrossBlankSourceLines() { assertArrayEquals(new int[] {0, 3}, lineFromTo); } + + @Test + void shouldReturnAllMatchingContextStartLines() throws Exception { + List matches = FuzzyContextSearcher.fuzzySearchContextMatches( + List.of("before", "target", "after", "target", "after"), + List.of("target", "after"), + 0); + + assertEquals(List.of(1, 3), matches); + } + + @Test + void shouldReturnOneMatchingContextStartLine() throws Exception { + List matches = FuzzyContextSearcher.fuzzySearchContextMatches( + List.of("before", "target", "after"), + List.of("target", "after"), + 0); + + assertEquals(List.of(1), matches); + } + + @Test + void shouldReturnNoMatchingContextStartLines() throws Exception { + List matches = FuzzyContextSearcher.fuzzySearchContextMatches( + List.of("before", "after"), + List.of("target", "after"), + 0); + + assertEquals(List.of(), matches); + } + + @Test + void shouldKeepPhysicalStartWhenContextBeginsWithBlankLine() throws Exception { + List matches = FuzzyContextSearcher.fuzzySearchContextMatches( + List.of("header", "", "target", "after"), + List.of("", "target", "after"), + 0); + + assertEquals(List.of(1), matches); + } + + @Test + void shouldMatchContextAcrossBlankSourceLines() throws Exception { + List matches = FuzzyContextSearcher.fuzzySearchContextMatches( + List.of("header", "target", "", "", "after", "target", "", "after"), + List.of("target", "after"), + 0); + + assertEquals(List.of(1, 5), matches); + } } \ No newline at end of file diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java index 5a0c876ef9e..f0c56048b13 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java @@ -80,6 +80,7 @@ public class AviatorSSCAuditCommand extends AbstractSSCJsonNodeOutputCommand imp @ArgGroup(exclusive = true, multiplicity = "0..1") private QuotaHandlingArgGroup quotaHandlingArgGroup = new QuotaHandlingArgGroup(); @Option(names = {"--test-exceeding-quota"}) private boolean testExceedingQuota; @Option(names = {"--default-quota-fallback"}) private boolean defaultQuotaFallback; + @Option(names = {"--force-reaudit"}) private boolean forceReaudit; @Mixin private SourceEncodingsMixin sourceEncodingsMixin; private static final Logger LOG = LoggerFactory.getLogger(AviatorSSCAuditCommand.class); private Long checkedQuotaBefore; @@ -102,7 +103,8 @@ public JsonNode getJsonNode(UnirestInstance unirest) { refreshMetricsIfNeeded(unirest, av, logger); - long auditableIssueCount = AviatorSSCAuditHelper.getAuditableIssueCount(unirest, av, logger, isNoFilterSet(), getFilterSetTitleOrId(), folderNames); + long auditableIssueCount = AviatorSSCAuditHelper.getAuditableIssueCount( + unirest, av, logger, isNoFilterSet(), getFilterSetTitleOrId(), folderNames, forceReaudit); if (auditableIssueCount == 0) { logger.progress("Audit skipped - no auditable issues found matching the specified filters."); ObjectNode result = AviatorSSCAuditHelper.buildResultNode(av, null, "SKIPPED"); @@ -238,7 +240,7 @@ private JsonNode evaluateQuota(UnirestInstance unirest, SSCAppVersionDescriptor logger.progress("Warning: Could not retrieve quota for '%s', proceeding with audit.", effectiveAppName); } else if (availableQuota >= 0 && auditableIssueCount > availableQuota) { checkedQuotaBefore = availableQuota; - var topCategories = AviatorSSCAuditHelper.getTopUnauditedCategories(unirest, av, logger, 10); + var topCategories = AviatorSSCAuditHelper.getTopUnauditedCategories(unirest, av, logger, 10, forceReaudit); String detailedMessage = AviatorSSCAuditHelper.formatQuotaExceededMessage( av, auditableIssueCount, availableQuota, topCategories); LOG.info(detailedMessage); @@ -277,6 +279,7 @@ private JsonNode processFpr(UnirestInstance unirest, SSCAppVersionDescriptor av, .noFilterSet(isNoFilterSet()) .folderNames(folderNames) .folderPriorityOrder(getFolderPriorityOrder()) + .forceReaudit(forceReaudit) .sourceDecoder(sourceEncodingsMixin.getSourceDecoder()) .build()); } catch (Exception e) { diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java index e9ce3228879..bf0be506133 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java @@ -213,69 +213,75 @@ public static String getProgressMessage(FPRAuditResult auditResult) { } /** - * Queries SSC to get the number of auditable issues for a given application version. + * Coarse SSC download gate. Force re-audit counts Aviator-processed issues even when + * SSC marks them audited; suppressed issues and human-audited issues Aviator never + * processed stay excluded. Last TagHistory writer is applied later from the FPR. */ - public static long getAuditableIssueCount(UnirestInstance unirest, SSCAppVersionDescriptor av, AviatorLoggerImpl logger, boolean noFilterSet, String filterSetTitleOrId, List folderNames) { + public static long getAuditableIssueCount(UnirestInstance unirest, SSCAppVersionDescriptor av, AviatorLoggerImpl logger, + boolean noFilterSet, String filterSetTitleOrId, List folderNames, boolean forceReaudit) { logger.progress("Status: Checking for auditable issues..."); + LOG.debug("Starting auditable issue count for SSC version {} (application='{}', version='{}') with pageLimit={}, noFilterSet={}, forceReaudit={}", + av.getVersionId(), av.getApplicationName(), av.getVersionName(), PAGE_LIMIT, noFilterSet, forceReaudit); - LOG.debug("Starting auditable issue count for SSC version {} (application='{}', version='{}') with pageLimit={}, noFilterSet={}", - av.getVersionId(), av.getApplicationName(), av.getVersionName(), PAGE_LIMIT, noFilterSet); + AuditableIssueCountQuery query = resolveAuditableIssueCountQuery( + unirest, av, logger, noFilterSet, filterSetTitleOrId, folderNames, forceReaudit); + return countEligibleIssues(query, logger); + } + + private record AuditableIssueCountQuery(UnirestInstance unirest, SSCAppVersionDescriptor av, boolean noFilterSet, + String filterSetGuid, String folderFilter, boolean forceReaudit) {} + private static AuditableIssueCountQuery resolveAuditableIssueCountQuery(UnirestInstance unirest, SSCAppVersionDescriptor av, + AviatorLoggerImpl logger, boolean noFilterSet, String filterSetTitleOrId, List folderNames, + boolean forceReaudit) { String effectiveFilterSetTitleOrId = filterSetTitleOrId != null && !filterSetTitleOrId.isBlank() ? filterSetTitleOrId : null; boolean effectiveNoFilterSet = noFilterSet && effectiveFilterSetTitleOrId == null; + SSCIssueFilterSetDescriptor filterSetDescriptor = resolveFilterSet( + unirest, av, logger, effectiveNoFilterSet, effectiveFilterSetTitleOrId); + String filterSetGuid = filterSetDescriptor != null ? filterSetDescriptor.getGuid() : null; + String folderFilter = resolveFolderFilter(logger, av, effectiveNoFilterSet, filterSetDescriptor, folderNames); + return new AuditableIssueCountQuery(unirest, av, effectiveNoFilterSet, filterSetGuid, folderFilter, forceReaudit); + } - // Apply filter set if specified - SSCIssueFilterSetDescriptor filterSetDescriptor = null; - String filterSetGuid = null; - if (!effectiveNoFilterSet) { - SSCIssueFilterSetHelper filterSetHelper = new SSCIssueFilterSetHelper(unirest, av.getVersionId()); - filterSetDescriptor = filterSetHelper.getDescriptorByTitleOrId(effectiveFilterSetTitleOrId, false); - if (filterSetDescriptor != null) { - logger.progress("Status: Applying filter set '%s' for issue count check", filterSetDescriptor.getTitle()); - filterSetGuid = filterSetDescriptor.getGuid(); - LOG.debug("Applied SSC filter set '{}' with guid {} while counting auditable issues for version {}", - filterSetDescriptor.getTitle(), filterSetGuid, av.getVersionId()); - } else { - LOG.debug("No SSC filter set resolved from options while counting auditable issues for version {}", - av.getVersionId()); - } + private static SSCIssueFilterSetDescriptor resolveFilterSet(UnirestInstance unirest, SSCAppVersionDescriptor av, + AviatorLoggerImpl logger, boolean noFilterSet, String filterSetTitleOrId) { + if (noFilterSet) { + return null; } - - // Apply folder filter if specified - String folderFilter = null; - if (folderNames != null && !folderNames.isEmpty()) { - folderFilter = getFolderFilter(effectiveNoFilterSet, filterSetDescriptor, folderNames); - logger.progress("Status: Applying folder filter for: %s", String.join(", ", folderNames)); - LOG.debug("Applied folder filter '{}' for folders {} while counting auditable issues for version {}", - folderFilter, folderNames, av.getVersionId()); + SSCIssueFilterSetHelper filterSetHelper = new SSCIssueFilterSetHelper(unirest, av.getVersionId()); + SSCIssueFilterSetDescriptor filterSetDescriptor = filterSetHelper.getDescriptorByTitleOrId(filterSetTitleOrId, false); + if (filterSetDescriptor != null) { + logger.progress("Status: Applying filter set '%s' for issue count check", filterSetDescriptor.getTitle()); + LOG.debug("Applied SSC filter set '{}' with guid {} while counting auditable issues for version {}", + filterSetDescriptor.getTitle(), filterSetDescriptor.getGuid(), av.getVersionId()); + } else { + LOG.debug("No SSC filter set resolved from options while counting auditable issues for version {}", + av.getVersionId()); } + return filterSetDescriptor; + } - long totalAuditableCount = 0; - int start = 0; - long totalFromServer = -1; + private static String resolveFolderFilter(AviatorLoggerImpl logger, SSCAppVersionDescriptor av, boolean noFilterSet, + SSCIssueFilterSetDescriptor filterSetDescriptor, List folderNames) { + if (folderNames == null || folderNames.isEmpty()) { + return null; + } + String folderFilter = getFolderFilter(noFilterSet, filterSetDescriptor, folderNames); + logger.progress("Status: Applying folder filter for: %s", String.join(", ", folderNames)); + LOG.debug("Applied folder filter '{}' for folders {} while counting auditable issues for version {}", + folderFilter, folderNames, av.getVersionId()); + return folderFilter; + } + private static long countEligibleIssues(AuditableIssueCountQuery query, AviatorLoggerImpl logger) { try { + long totalAuditableCount = 0; + int start = 0; + long totalFromServer = -1; do { - GetRequest request = unirest.get(SSCUrls.PROJECT_VERSION_ISSUES(av.getVersionId())) - .queryString("limit", PAGE_LIMIT) - .queryString("embed", "auditValues") - .queryString("qm", "issues") - .queryString("q", "audited:false") - .queryString("start", start); - if (effectiveNoFilterSet) { - request.queryString("showhidden", "true"); - } - if (filterSetGuid != null) { - request.queryString("filterset", filterSetGuid); - } - if (folderFilter != null) { - request.queryString("filter", folderFilter); - } - LOG.debug("Requesting SSC issues page for version {} with start={} and limit={}", - av.getVersionId(), start, PAGE_LIMIT); - JsonNode response = request.asObject(JsonNode.class).getBody(); + JsonNode response = fetchAuditableIssuesPage(query, start); if (response == null || !response.has("data")) { LOG.warn("Invalid response received from issue check; proceeding with FPR download."); logger.progress("WARN: Invalid response from issue check. Proceeding with FPR download."); @@ -284,27 +290,23 @@ public static long getAuditableIssueCount(UnirestInstance unirest, SSCAppVersion if (totalFromServer == -1) { totalFromServer = response.get("count").asLong(0); LOG.debug("SSC reported {} total issues matching the initial auditable count query for version {}", - totalFromServer, av.getVersionId()); + totalFromServer, query.av().getVersionId()); } - ArrayNode issues = (ArrayNode) response.get("data"); - if (issues != null && !issues.isEmpty()) { - long auditableOnPage = JsonHelper.stream(issues) - .filter(issue -> !isProcessedByAviator(issue)) - .count(); - totalAuditableCount += auditableOnPage; - LOG.debug("Processed SSC issues page for version {}: pageStart={}, pageSize={}, auditableOnPage={}, cumulativeAuditableCount={}, totalFromServer={}", - av.getVersionId(), start, issues.size(), auditableOnPage, totalAuditableCount, totalFromServer); - start += issues.size(); - } else { + if (issues == null || issues.isEmpty()) { LOG.debug("SSC returned no more issues for version {} at start={}; stopping pagination with cumulativeAuditableCount={} and totalFromServer={}", - av.getVersionId(), start, totalAuditableCount, totalFromServer); - break; // No more issues + query.av().getVersionId(), start, totalAuditableCount, totalFromServer); + break; } + long auditableOnPage = countEligibleOnPage(issues, query.forceReaudit()); + totalAuditableCount += auditableOnPage; + LOG.debug("Processed SSC issues page for version {}: pageStart={}, pageSize={}, auditableOnPage={}, cumulativeAuditableCount={}, totalFromServer={}", + query.av().getVersionId(), start, issues.size(), auditableOnPage, totalAuditableCount, totalFromServer); + start += issues.size(); } while (start < totalFromServer); LOG.debug("Completed auditable issue count for version {}: totalAuditableCount={}, totalFromServer={}", - av.getVersionId(), totalAuditableCount, totalFromServer); + query.av().getVersionId(), totalAuditableCount, totalFromServer); logger.progress("Status: Found %d auditable issues.", totalAuditableCount); return totalAuditableCount; } catch (UnexpectedHttpResponseException e) { @@ -314,6 +316,52 @@ public static long getAuditableIssueCount(UnirestInstance unirest, SSCAppVersion } } + private static JsonNode fetchAuditableIssuesPage(AuditableIssueCountQuery query, int start) { + GetRequest request = query.unirest().get(SSCUrls.PROJECT_VERSION_ISSUES(query.av().getVersionId())) + .queryString("limit", PAGE_LIMIT) + .queryString("embed", "auditValues") + .queryString("qm", "issues") + .queryString("start", start); + // Normal audit only needs unaudited issues. Force re-audit must also + // retrieve Aviator-processed issues, which SSC marks audited after Analysis is set. + if (!query.forceReaudit()) { + request.queryString("q", "audited:false"); + } + if (query.noFilterSet()) { + request.queryString("showhidden", "true"); + } + if (query.filterSetGuid() != null) { + request.queryString("filterset", query.filterSetGuid()); + } + if (query.folderFilter() != null) { + request.queryString("filter", query.folderFilter()); + } + LOG.debug("Requesting SSC issues page for version {} with start={} and limit={}", + query.av().getVersionId(), start, PAGE_LIMIT); + return request.asObject(JsonNode.class).getBody(); + } + + private static long countEligibleOnPage(ArrayNode issues, boolean forceReaudit) { + return JsonHelper.stream(issues) + .filter(issue -> isEligibleForRequestedAudit(issue, forceReaudit)) + .count(); + } + + /** + * Force re-audit includes previously processed Aviator issues even when SSC + * already marks them audited. Suppressed issues and human-audited issues that + * Aviator never processed stay out. Normal audit still excludes processed issues. + */ + private static boolean isEligibleForRequestedAudit(JsonNode issue, boolean forceReaudit) { + if (issue.path("suppressed").asBoolean(false)) { + return false; + } + if (isProcessedByAviator(issue)) { + return forceReaudit; + } + return !issue.path("audited").asBoolean(false); + } + /** * Checks if an issue has the 'Aviator status' custom tag set. */ @@ -425,18 +473,17 @@ public static long getDefaultQuota(String aviatorUrl, String aviatorToken, } /** - * Returns the top N SAST categories ordered by truly-unaudited issue count (descending). - * Uses the SSC issueGroups API with: - * - groupingtype = dynamically resolved "Category" GUID - * - filter = dynamically resolved "Aviator status:Not Set" technical filter - * Each entry contains "categoryName" and "unauditedCount". + * Returns the top N SAST categories for the requested audit mode. + * Normal audit ranks by unaudited issues with Aviator status not set. + * Force re-audit ranks by visible issues, including Aviator-processed ones. + * Each entry contains "categoryName" and "unauditedCount" (eligible count for that mode). */ public static List> getTopUnauditedCategories( UnirestInstance unirest, SSCAppVersionDescriptor av, - AviatorLoggerImpl logger, int topN) { + AviatorLoggerImpl logger, int topN, boolean forceReaudit) { try { - return getTopUnauditedCategoriesInternal(unirest, av, logger, topN); + return getTopUnauditedCategoriesInternal(unirest, av, logger, topN, forceReaudit); } catch (Exception e) { LOG.warn("Failed to retrieve top unaudited categories for {}:{}: {}", av.getApplicationName(), av.getVersionName(), e.getMessage()); @@ -447,7 +494,7 @@ public static List> getTopUnauditedCategories( private static List> getTopUnauditedCategoriesInternal( UnirestInstance unirest, SSCAppVersionDescriptor av, - AviatorLoggerImpl logger, int topN) { + AviatorLoggerImpl logger, int topN, boolean forceReaudit) { String versionId = av.getVersionId(); @@ -458,12 +505,14 @@ private static List> getTopUnauditedCategoriesInternal( // Resolve "Aviator status:Not Set" filter dynamically String aviatorStatusFilter = null; - try { - SSCIssueFilterHelper filterHelper = new SSCIssueFilterHelper(unirest, versionId); - aviatorStatusFilter = filterHelper.getFilter("Aviator status:Not Set"); - } catch (FcliSimpleException e) { - // Tag doesn't exist on this version — all issues are unprocessed - LOG.debug("Aviator status tag not found for version {}. All issues considered unprocessed.", versionId); + if (!forceReaudit) { + try { + SSCIssueFilterHelper filterHelper = new SSCIssueFilterHelper(unirest, versionId); + aviatorStatusFilter = filterHelper.getFilter("Aviator status:Not Set"); + } catch (FcliSimpleException e) { + // Tag doesn't exist on this version — all issues are unprocessed + LOG.debug("Aviator status tag not found for version {}. All issues considered unprocessed.", versionId); + } } // Call issueGroups API @@ -479,20 +528,22 @@ private static List> getTopUnauditedCategoriesInternal( JsonNode response = request.asObject(JsonNode.class).getBody(); ArrayNode groups = (ArrayNode) response.get("data"); - // Calculate truly-unaudited count per category and sort descending + // Rank categories by issues eligible for the requested audit mode. List> categories = new ArrayList<>(); if (groups != null) { for (JsonNode group : groups) { String name = group.path("id").asText("Unknown"); long visibleCount = group.path("visibleCount").asLong(0); long auditedCount = group.path("auditedCount").asLong(0); - long unaudited = visibleCount - auditedCount; - if (unaudited > 0) { + // Force re-audit sends Aviator-processed (already audited) issues, so + // visible count is the quota-relevant size for the category. + long eligibleCount = forceReaudit ? visibleCount : visibleCount - auditedCount; + if (eligibleCount > 0) { Map entry = new LinkedHashMap<>(); entry.put("categoryName", name); entry.put("totalIssues", visibleCount); entry.put("auditedIssues", auditedCount); - entry.put("unauditedCount", unaudited); + entry.put("unauditedCount", eligibleCount); categories.add(entry); } } diff --git a/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties b/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties index 7a2c64f68e3..907990c0223 100644 --- a/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties +++ b/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties @@ -128,6 +128,7 @@ fcli.aviator.ssc.audit.folder = Filter issues by a comma-separated list of speci fcli.aviator.ssc.audit.skip-if-exceeding-quota = Skip audit if the number of open issues exceeds the available Fortify Remediation Aviator quota. When skipped, a summary with top unaudited categories is shown. fcli.aviator.ssc.audit.test-exceeding-quota = Check whether the number of open issues exceeds the available Fortify Remediation Aviator quota and report the result without performing an audit. fcli.aviator.ssc.audit.default-quota-fallback = (Internal) When the Fortify Aviator application does not exist, use the tenant default quota instead of reporting app not found. Used by bulk audit. +fcli.aviator.ssc.audit.force-reaudit = Re-audit issues previously processed by Aviator, while preserving suppressed and manually triaged issues. fcli.aviator.ssc.audit.folder-priority-order = Custom priority order for folder-based filtering when quota is exceeded (comma-separated, highest priority first). Example: Critical,High,Medium,Low. If not specified, uses default priority order. fcli.aviator.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files. Use FPR to try the source encoding recorded in audit.fvdl. When writing remediated files, the accepted encoding is used. Default value: ${DEFAULT-VALUE}. fcli.aviator.ssc.audit.refresh = By default, this command will refresh the source application version's metrics when copying from it. \ diff --git a/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java b/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java index d8988e8a860..c1041fd5a75 100644 --- a/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java +++ b/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java @@ -17,6 +17,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collections; import java.util.Map; @@ -52,6 +53,14 @@ void testAllowsNoFilterSetOption() { assertNull(cmd.getFilterSetTitleOrId()); } + @Test + void testAllowsForceReauditOption() throws Exception { + var cmd = parse("--force-reaudit"); + Field field = AviatorSSCAuditCommand.class.getDeclaredField("forceReaudit"); + field.setAccessible(true); + assertTrue((boolean) field.get(cmd)); + } + @Test void reportsDecodeSkippedIssuesAsNotSubmitted() { ObjectNode result = JsonHelper.getObjectMapper().createObjectNode(); diff --git a/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelperTest.java b/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelperTest.java new file mode 100644 index 00000000000..c1a3f0f6122 --- /dev/null +++ b/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelperTest.java @@ -0,0 +1,252 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.ssc.helper; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.aviator.config.AviatorLoggerImpl; +import com.fortify.cli.common.json.JsonHelper; +import com.fortify.cli.common.progress.helper.IProgressWriter; +import com.fortify.cli.common.rest.unirest.UnirestHelper; +import com.fortify.cli.common.rest.unirest.config.UnirestJsonHeaderConfigurer; +import com.fortify.cli.ssc.appversion.helper.SSCAppVersionDescriptor; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import kong.unirest.UnirestInstance; + +class AviatorSSCAuditHelperTest { + + @Test + void excludesPreviouslyProcessedIssuesFromNormalPreflightCount() throws Exception { + try (var server = new TestSscServer(processedIssue(), unprocessedIssue()); + var unirest = newUnirest(server)) { + assertEquals(1, AviatorSSCAuditHelper.getAuditableIssueCount( + unirest, appVersion(), logger(), true, null, null, false)); + assertTrue(server.getLastIssueQuery().contains("q=audited:false")); + } + } + + @Test + void forceReauditPreflightIncludesPreviouslyProcessedIssues() throws Exception { + try (var server = new TestSscServer(processedIssue(), unprocessedIssue()); + var unirest = newUnirest(server)) { + assertEquals(2, AviatorSSCAuditHelper.getAuditableIssueCount( + unirest, appVersion(), logger(), true, null, null, true)); + assertFalse(server.getLastIssueQuery().contains("q=audited:false")); + } + } + + @Test + void forceReauditPreflightDoesNotSkipAnAllProcessedApplicationVersion() throws Exception { + try (var server = new TestSscServer(processedIssue()); + var unirest = newUnirest(server)) { + assertEquals(1, AviatorSSCAuditHelper.getAuditableIssueCount( + unirest, appVersion(), logger(), true, null, null, true)); + assertFalse(server.getLastIssueQuery().contains("q=audited:false")); + } + } + + @Test + void forceReauditPreflightExcludesHumanAuditedIssuesThatAviatorNeverProcessed() throws Exception { + try (var server = new TestSscServer(processedIssue(), unprocessedIssue(), humanAuditedIssue()); + var unirest = newUnirest(server)) { + assertEquals(2, AviatorSSCAuditHelper.getAuditableIssueCount( + unirest, appVersion(), logger(), true, null, null, true)); + assertFalse(server.getLastIssueQuery().contains("q=audited:false")); + } + } + + @Test + void forceReauditPreflightExcludesSuppressedAviatorIssues() throws Exception { + try (var server = new TestSscServer(processedIssue(), suppressedProcessedIssue()); + var unirest = newUnirest(server)) { + assertEquals(1, AviatorSSCAuditHelper.getAuditableIssueCount( + unirest, appVersion(), logger(), true, null, null, true)); + } + } + + @Test + void forceReauditCategoryBreakdownIncludesFullyAuditedCategories() throws Exception { + try (var server = new TestSscServer(processedIssue()); + var unirest = newUnirest(server)) { + List> categories = AviatorSSCAuditHelper.getTopUnauditedCategories( + unirest, appVersion(), logger(), 10, true); + + assertEquals(1, categories.size()); + assertEquals("Category A", categories.get(0).get("categoryName")); + assertEquals(1L, categories.get(0).get("unauditedCount")); + assertEquals(1, server.getSelectorSetRequestCount()); + } + } + + private static SSCAppVersionDescriptor appVersion() { + var descriptor = new SSCAppVersionDescriptor(); + descriptor.setVersionId("42"); + descriptor.setApplicationName("TestApp"); + descriptor.setVersionName("1.0"); + return descriptor; + } + + private static AviatorLoggerImpl logger() { + return new AviatorLoggerImpl(new NoOpProgressWriter()); + } + + private static UnirestInstance newUnirest(TestSscServer server) { + return UnirestHelper.createUnirestInstance(unirest -> { + UnirestJsonHeaderConfigurer.configure(unirest); + unirest.config().defaultBaseUrl(server.getBaseUrl()); + }); + } + + private static ObjectNode processedIssue() { + ObjectNode issue = JsonHelper.getObjectMapper().createObjectNode(); + issue.put("audited", true); + ArrayNode auditValues = issue.putObject("_embed").putArray("auditValues"); + auditValues.addObject() + .put("customTagGuid", AviatorSSCTagDefs.AVIATOR_STATUS_TAG.getGuid()) + .put("customTagIndex", 0); + return issue; + } + + private static ObjectNode unprocessedIssue() { + ObjectNode issue = JsonHelper.getObjectMapper().createObjectNode(); + issue.put("audited", false); + issue.putObject("_embed").putArray("auditValues"); + return issue; + } + + private static ObjectNode humanAuditedIssue() { + ObjectNode issue = JsonHelper.getObjectMapper().createObjectNode(); + issue.put("audited", true); + issue.putObject("_embed").putArray("auditValues"); + return issue; + } + + private static ObjectNode suppressedProcessedIssue() { + ObjectNode issue = processedIssue(); + issue.put("suppressed", true); + return issue; + } + + private static final class TestSscServer implements AutoCloseable { + private final HttpServer server; + private final ArrayNode issues = JsonHelper.getObjectMapper().createArrayNode(); + private int selectorSetRequestCount; + private String lastIssueQuery = ""; + + private TestSscServer(ObjectNode... issueNodes) throws IOException { + for (ObjectNode issue : issueNodes) { + issues.add(issue); + } + server = HttpServer.create(new InetSocketAddress(0), 0); + server.createContext("/api/v1/projectVersions/42/issues", this::handleIssues); + server.createContext("/api/v1/projectVersions/42/issueSelectorSet", this::handleSelectorSet); + server.createContext("/api/v1/projectVersions/42/issueGroups", this::handleIssueGroups); + server.start(); + } + + private String getBaseUrl() { + return "http://localhost:" + server.getAddress().getPort(); + } + + private void handleIssues(HttpExchange exchange) throws IOException { + lastIssueQuery = URLDecoder.decode(exchange.getRequestURI().getRawQuery(), StandardCharsets.UTF_8); + ArrayNode matchingIssues = JsonHelper.getObjectMapper().createArrayNode(); + boolean unauditedOnly = lastIssueQuery.contains("q=audited:false"); + for (JsonNode issue : issues) { + if (unauditedOnly && issue.path("audited").asBoolean(false)) { + continue; + } + matchingIssues.add(issue); + } + byte[] response = JsonHelper.getObjectMapper().createObjectNode() + .put("count", matchingIssues.size()) + .set("data", matchingIssues) + .toString() + .getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, response.length); + try (var outputStream = exchange.getResponseBody()) { + outputStream.write(response); + } + } + + private void handleSelectorSet(HttpExchange exchange) throws IOException { + selectorSetRequestCount++; + ObjectNode data = JsonHelper.getObjectMapper().createObjectNode(); + data.putArray("filterBySet"); + data.putArray("groupBySet").addObject() + .put("guid", "category-guid") + .put("displayName", "Category"); + writeJson(exchange, JsonHelper.getObjectMapper().createObjectNode().set("data", data)); + } + + private void handleIssueGroups(HttpExchange exchange) throws IOException { + ObjectNode group = JsonHelper.getObjectMapper().createObjectNode() + .put("id", "Category A") + .put("visibleCount", 1) + .put("auditedCount", 1); + ArrayNode data = JsonHelper.getObjectMapper().createArrayNode().add(group); + writeJson(exchange, JsonHelper.getObjectMapper().createObjectNode().set("data", data)); + } + + private void writeJson(HttpExchange exchange, ObjectNode responseNode) throws IOException { + byte[] response = responseNode.toString().getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, response.length); + try (var outputStream = exchange.getResponseBody()) { + outputStream.write(response); + } + } + + private String getLastIssueQuery() { + return lastIssueQuery; + } + + private int getSelectorSetRequestCount() { + return selectorSetRequestCount; + } + + @Override + public void close() { + server.stop(0); + } + } + + private static final class NoOpProgressWriter implements IProgressWriter { + @Override public boolean isMultiLineSupported() { return false; } + @Override public void writeProgress(String message, Object... args) {} + @Override public void writeInfo(String message, Object... args) {} + @Override public void writeInfoWithException(String message, Throwable cause, Object... args) {} + @Override public void writeWarning(String message, Object... args) {} + @Override public void writeWarningWithException(String message, Throwable cause, Object... args) {} + @Override public void clearProgress() {} + @Override public void close() {} + @Override public String type() { return "test"; } + } +} From e46ed96bff195dc868d733e5ed59038c5c4f5f46 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:21:30 +0000 Subject: [PATCH 09/37] chore(rel/v3.x): release 3.25.0 --- .release-please-manifest.json | 2 +- CHANGELOG.md | 12 ++++++++++++ version.txt | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 6973dab8638..c3511605211 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "3.24.0" + ".": "3.25.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 53e8bbefaed..d18c08fc2ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [3.25.0](https://github.com/fortify/fcli/compare/v3.24.0...v3.25.0) (2026-08-31) + + +### Features + +* `fcli aviator ssc audit`: Add `--force-reaudit` to re-audit Aviator-processed issues without overwriting human triage ([840ee90](https://github.com/fortify/fcli/commit/840ee90beed814507f06c01c0e456f1b610f4234)) + + +### Bug Fixes + +* `fcli aviator ssc apply-remediations`: Skip remediations when source context matches multiple locations ([840ee90](https://github.com/fortify/fcli/commit/840ee90beed814507f06c01c0e456f1b610f4234)) + ## [3.24.0](https://github.com/fortify/fcli/compare/v3.23.3...v3.24.0) (2026-08-21) diff --git a/version.txt b/version.txt index 954e2288218..0914443131f 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.24.0 +3.25.0 From 56a66a68b2bfc45527292d091adb4b2483b6716c Mon Sep 17 00:00:00 2001 From: Ankit Date: Mon, 31 Aug 2026 15:00:07 +0530 Subject: [PATCH 10/37] chore: Add null check for auditIssue in isResultTagSet method (#1084) Co-authored-by: Ankit Rathod --- .../main/java/com/fortify/cli/aviator/audit/IssueAuditor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java index 40bb0d4c4e5..79e400cd0f1 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java @@ -323,6 +323,9 @@ private boolean isAviatorWork(AuditIssue auditIssue) { } private boolean isResultTagSet(AuditIssue auditIssue, String tagId) { + if (auditIssue == null) { + return false; + } Map tags = auditIssue.getTags(); return tags != null && !StringUtil.isPendingReviewValue(tags.get(tagId)); } From f4a46aa2d04bbcbfd6c865888c52b9ff519f9cce Mon Sep 17 00:00:00 2001 From: umadevis1 Date: Thu, 17 Sep 2026 20:48:05 +0530 Subject: [PATCH 11/37] chore: Merge P/umadevis1/improve remediations (#1089) fix: `fcli aviator ssc apply-remediations`: More accurate application of auto-remediations fix: `fcli aviator ssc apply-remediations`: Improve handling of overlapping remediations fix: `fcli fod aviator apply-remediations`: More accurate application of auto-remediations fix: `fcli fod aviator apply-remediations`: Improve handling of overlapping remediations Co-authored-by: Umadevi Santhanam Co-authored-by: Frans van Buul --- .../ApplyAutoRemediationOnSource.java | 11 +- .../aviator/fpr/processor/AuditProcessor.java | 13 +- .../fpr/processor/RemediationProcessor.java | 580 ----------- .../fpr/remediation/RemediationProcessor.java | 334 ++++++ .../aviator/fpr/remediation/SkipReason.java | 42 + .../applier/AppliedChangeResult.java | 15 + .../applier/FuzzyAnchorLocator.java | 81 ++ .../fpr/remediation/applier/LineRange.java | 17 + .../applier/RemediationApplier.java | 308 ++++++ .../classifier/AppliedChangeLedger.java | 106 ++ .../classifier/HunkClassifier.java | 95 ++ .../classifier/PendingAppliedChange.java | 19 + .../exception/RemediationCommitException.java | 33 + .../RollbackRemediationException.java | 23 + .../exception/SkipRemediationException.java | 37 + .../fpr/remediation/model/AppliedChange.java | 118 +++ .../fpr/remediation/model/FileChange.java | 54 + .../aviator/fpr/remediation/model/Hunk.java | 177 ++++ .../fpr/remediation/model/HunkOutcome.java | 18 + .../fpr/remediation/model/Remediation.java | 69 ++ .../model/RemediationDocument.java | 27 + .../fpr/remediation/model/RemediationKey.java | 25 + .../remediation/model/RemediationMetric.java | 56 + .../fpr/remediation/model/RequiredFields.java | 58 ++ .../writer/FileWriteCoordinator.java | 219 ++++ .../remediation/writer/PendingFileWrite.java | 20 + .../writer/PreparedFileChanges.java | 23 + .../remediation/writer/RollbackFileWrite.java | 18 + .../remediation/writer/SourceFileContent.java | 40 + .../RemediationDocumentMapper.java | 96 ++ .../xmlprocessor/RemediationXmlReader.java | 51 + .../fortify/cli/aviator/util/FileUtil.java | 55 +- .../fortify/cli/aviator/util/FprHandle.java | 6 +- .../aviator/util/FuzzyContextSearcher.java | 15 +- .../processor/RemediationProcessorTest.java | 957 +++++++++++++++++- .../remediation/model/AppliedChangeTest.java | 37 + .../cli/aviator/util/FileUtilTest.java | 74 ++ .../cli/aviator/util/FprHandleTest.java | 40 + .../util/FuzzyContextSearcherTest.java | 13 +- .../AviatorSSCApplyRemediationsCommand.java | 39 +- .../AviatorSSCApplyRemediationsHelper.java | 52 +- .../aviator/i18n/AviatorMessages.properties | 4 +- ...rSSCApplyRemediationsCommandOrderTest.java | 321 ++++++ .../FoDAviatorApplyRemediationsCommand.java | 6 +- .../AviatorFoDApplyRemediationsHelper.java | 35 +- .../cli/fod/i18n/FoDMessages.properties | 2 +- 46 files changed, 3741 insertions(+), 698 deletions(-) delete mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/RemediationProcessor.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/SkipReason.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/AppliedChangeResult.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/FuzzyAnchorLocator.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/LineRange.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/RemediationApplier.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/classifier/AppliedChangeLedger.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/classifier/HunkClassifier.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/classifier/PendingAppliedChange.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/exception/RemediationCommitException.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/exception/RollbackRemediationException.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/exception/SkipRemediationException.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/AppliedChange.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/FileChange.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/Hunk.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/HunkOutcome.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/Remediation.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RemediationDocument.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RemediationKey.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RemediationMetric.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RequiredFields.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/FileWriteCoordinator.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/PendingFileWrite.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/PreparedFileChanges.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/RollbackFileWrite.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/SourceFileContent.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/xmlprocessor/RemediationDocumentMapper.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/xmlprocessor/RemediationXmlReader.java create mode 100644 fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/remediation/model/AppliedChangeTest.java create mode 100644 fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/util/FileUtilTest.java create mode 100644 fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommandOrderTest.java diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java index 23b77f0119a..89ce88c2b97 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java @@ -19,9 +19,8 @@ import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; -import com.fortify.cli.aviator.config.IAviatorLogger; -import com.fortify.cli.aviator.fpr.processor.RemediationProcessor; -import com.fortify.cli.aviator.fpr.processor.RemediationProcessor.RemediationMetric; +import com.fortify.cli.aviator.fpr.remediation.RemediationProcessor; +import com.fortify.cli.aviator.fpr.remediation.model.RemediationMetric; import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.util.FprHandle; @@ -30,13 +29,13 @@ public class ApplyAutoRemediationOnSource { private static final Logger LOG = LoggerFactory.getLogger(ApplyAutoRemediationOnSource.class); - public static RemediationMetric applyRemediations(FprHandle fprHandle, String sourceCodeDirectory, IAviatorLogger logger) + public static RemediationMetric applyRemediations(FprHandle fprHandle, String sourceCodeDirectory) throws AviatorSimpleException, AviatorTechnicalException { - return applyRemediations(fprHandle, sourceCodeDirectory, SourceDecoders.defaults(), logger); + return applyRemediations(fprHandle, sourceCodeDirectory, SourceDecoders.defaults()); } public static RemediationMetric applyRemediations(FprHandle fprHandle, String sourceCodeDirectory, - ISourceDecoder sourceDecoder, IAviatorLogger logger) + ISourceDecoder sourceDecoder) throws AviatorSimpleException, AviatorTechnicalException { LOG.info("Starting apply auto-remediation process for file: {}", fprHandle.getFprPath()); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java index 11a79e38abf..5707630dfc7 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java @@ -66,11 +66,13 @@ import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.util.Constants; +import com.fortify.cli.aviator.util.FileUtil; import com.fortify.cli.aviator.util.FprHandle; import lombok.Setter; + public class AuditProcessor { Logger logger = LoggerFactory.getLogger(AuditProcessor.class); @@ -938,7 +940,9 @@ private Document generateRemediationsXml(Map auditRespons changeElement.appendChild(originalCodeElement); Element newCodeElement = finalDoc.createElementNS(REMEDIATIONS_NAMESPACE_URI, "NewCode"); - newCodeElement.appendChild(finalDoc.createCDATASection(change.getReplaceWith() != null ? change.getReplaceWith() : "")); + String sanitizedNewCode = FileUtil.stripSyntheticLineMarkers( + change.getReplaceWith() != null ? change.getReplaceWith() : "", filename); + newCodeElement.appendChild(finalDoc.createCDATASection(sanitizedNewCode)); changeElement.appendChild(newCodeElement); final int CONTEXT_LINES = 3; @@ -1007,10 +1011,13 @@ private void recordSkipped(Map skippedByReason, RemediationSkip } private String calculateHashBase64(String content, String algorithm) { - if (content == null) return ""; try { MessageDigest md = MessageDigest.getInstance(algorithm); - byte[] digest = md.digest(content.getBytes(StandardCharsets.UTF_8)); + // hash the canonical form (LF-normalised, no trailing newline) so the apply + // side can reproduce the digest regardless of the OS that ran the audit or the + // file's trailing-newline state. + String canonical = FileUtil.canonicalizeForHash(content); + byte[] digest = md.digest(canonical.getBytes(StandardCharsets.UTF_8)); return Base64.getEncoder().encodeToString(digest); } catch (NoSuchAlgorithmException e) { throw new AviatorTechnicalException("Hashing algorithm not available: " + algorithm, e); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java deleted file mode 100644 index 2f717dd66b8..00000000000 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java +++ /dev/null @@ -1,580 +0,0 @@ -/* - * Copyright 2021-2026 Open Text. - * - * The only warranties for products and services of Open Text - * and its affiliates and licensors ("Open Text") are as may - * be set forth in the express warranty statements accompanying - * such products and services. Nothing herein should be construed - * as constituting an additional warranty. Open Text shall not be - * liable for technical or editorial errors or omissions contained - * herein. The information contained herein is subject to change - * without notice. - */ -package com.fortify.cli.aviator.fpr.processor; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Base64; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.zip.ZipFile; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; - -import com.fortify.cli.aviator._common.exception.AviatorSimpleException; -import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; -import com.fortify.cli.aviator.fpr.model.FVDLMetadata; -import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; -import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.DecodeResult; -import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.SourceDecodeException; -import com.fortify.cli.aviator.fpr.utils.SourceDecoders; -import com.fortify.cli.aviator.fpr.utils.SourceEncoder; -import com.fortify.cli.aviator.fpr.utils.SourceEncoder.SourceEncodeException; -import com.fortify.cli.aviator.util.FprHandle; -import com.fortify.cli.aviator.util.FuzzyContextSearcher; - -public class RemediationProcessor { - private static final Logger LOG = LoggerFactory.getLogger(RemediationProcessor.class); - private static final String NAMESPACE_URI = "xmlns://www.fortify.com/schema/remediations"; - - private final FprHandle fprHandle; - private final String sourceCodeDirectory; - private final ISourceDecoder sourceDecoder; - - public record RemediationMetric(int totalRemediations, int appliedRemediations, int skippedRemediations, Set modifiedFiles, - Map skippedByReason) { - public RemediationMetric(int totalRemediations, int appliedRemediations, int skippedRemediations, Set modifiedFiles) { - this(totalRemediations, appliedRemediations, skippedRemediations, modifiedFiles, Map.of()); - } - } - - private record SourceFileContent(String content, Charset charset, String encodingSource) {} - - private record PendingFileWrite(String filename, Path filePath, String content, Charset charset, String encodingSource, - byte[] updatedBytes) {} - - private record RollbackFileWrite(String filename, Path filePath, byte[] originalBytes) {} - - private enum SkipReason { - SOURCE_FILE_MISSING("Source file missing"), - SOURCE_FILE_OUTSIDE_SOURCE_DIR("Source file outside source directory"), - SOURCE_READ_FAILED("Source file read failed"), - SOURCE_DECODE_FAILED("Source file decode failed"), - REMEDIATION_DATA_INVALID("Remediation data invalid"), - REMEDIATION_LINE_RANGE_INVALID("Remediation line range invalid"), - SOURCE_CONTEXT_NOT_FOUND("Source context not found"), - SOURCE_CONTEXT_AMBIGUOUS("Source context matched multiple locations"), - ORIGINAL_CODE_NOT_FOUND("Original code not found"), - REMEDIATION_ENCODE_FAILED("Remediation encode failed"), - SOURCE_WRITE_FAILED("Source file write failed"), - NO_CHANGES("No file changes found"), - UNEXPECTED_ERROR("Unexpected remediation processing error"); - - private final String displayName; - - SkipReason(String displayName) { - this.displayName = displayName; - } - } - - private static class SkipRemediationException extends AviatorSimpleException { - private static final long serialVersionUID = 1L; - - private final SkipReason reason; - - SkipRemediationException(SkipReason reason, String message) { - super(message); - this.reason = reason; - } - - SkipRemediationException(SkipReason reason, String message, Throwable cause) { - super(message, cause); - this.reason = reason; - } - } - - private static class RemediationCommitException extends AviatorTechnicalException { - private static final long serialVersionUID = 1L; - - private final List rollbacks; - - RemediationCommitException(String message, Throwable cause, List rollbacks) { - super(message, cause); - this.rollbacks = rollbacks; - } - - List getRollbacks() { - return rollbacks; - } - } - - private static class RollbackRemediationException extends AviatorTechnicalException { - private static final long serialVersionUID = 1L; - - RollbackRemediationException(String message, Throwable cause) { - super(message, cause); - } - } - - public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory) { - this(fprHandle, sourceCodeDirectory, SourceDecoders.defaults()); - } - - public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory, ISourceDecoder sourceDecoder) { - this.fprHandle = fprHandle; - this.sourceCodeDirectory = sourceCodeDirectory; - this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); - } - - public RemediationMetric processRemediationXML() { - Path remediationPath = fprHandle.getPath("/remediations.xml"); - Document remediationDoc; - int totalRemediations; - int appliedRemediations; - Set modifiedFiles = new LinkedHashSet<>(); - Map skippedByReason = new LinkedHashMap<>(); - - // Sanitize and normalize the base source directory path once. - String trimmedSourceDir = sourceCodeDirectory.trim(); - if (trimmedSourceDir.length() > 1 && - ((trimmedSourceDir.startsWith("\"") && trimmedSourceDir.endsWith("\"")) || - (trimmedSourceDir.startsWith("'") && trimmedSourceDir.endsWith("'")))) { - trimmedSourceDir = trimmedSourceDir.substring(1, trimmedSourceDir.length() - 1); - } - final Path sourceBasePath = Paths.get(trimmedSourceDir).toAbsolutePath().normalize(); - LOG.debug("Applying remediations from {} to source directory {}", remediationPath, sourceBasePath); - final FVDLMetadata fvdlMetadata = loadFvdlMetadata(); - - try (InputStream remediationStream = Files.newInputStream(remediationPath)) { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setFeature("http://xml.org/sax/features/external-general-entities", false); - factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); - factory.setXIncludeAware(false); - factory.setExpandEntityReferences(false); - DocumentBuilder builder = factory.newDocumentBuilder(); - remediationDoc = builder.parse(remediationStream); - - NodeList remediationNodes = remediationDoc.getElementsByTagNameNS(NAMESPACE_URI, "Remediation"); - totalRemediations = remediationNodes.getLength(); - LOG.debug("Loaded {} remediation entries from {}", totalRemediations, remediationPath); - appliedRemediations = 0; - for (int i = 0; i < remediationNodes.getLength(); i++) { - Element remediation = (Element) remediationNodes.item(i); - if (processRemediation(remediation, sourceBasePath, fvdlMetadata, modifiedFiles, skippedByReason)) { - appliedRemediations++; - } - } - - } catch (ParserConfigurationException | SAXException | IOException e) { - LOG.error("Error parsing remediations.xml file: {}", remediationPath, e); - throw new AviatorTechnicalException("Error processing remediation.xml file.", e); - } catch (AviatorTechnicalException e) { - throw e; - } catch (Exception e) { - LOG.error("Unexpected error processing remediation.xml: {}", remediationPath, e); - throw new AviatorTechnicalException("Unexpected error processing remediations.xml.", e); - } - int skippedRemediations = totalRemediations - appliedRemediations; - LOG.info("Auto-remediation summary: total={}, applied={}, skipped={}", totalRemediations, appliedRemediations, skippedRemediations); - if (!skippedByReason.isEmpty()) { - LOG.info("Skipped remediations by reason: {}", formatSkippedReasons(skippedByReason)); - } - return new RemediationMetric(totalRemediations, appliedRemediations, skippedRemediations, modifiedFiles, skippedByReason); - } - - private boolean processRemediation(Element remediation, Path sourceBasePath, FVDLMetadata fvdlMetadata, - Set modifiedFiles, Map skippedByReason) { - String instanceId = remediation.getAttribute("instanceId"); - try { - Map pendingWrites = prepareFileChanges(remediation, sourceBasePath, fvdlMetadata); - if (pendingWrites.isEmpty()) { - recordSkipped(skippedByReason, SkipReason.NO_CHANGES.displayName); - return false; - } - try { - commitRemediationWrites(instanceId, pendingWrites, modifiedFiles); - return true; - } catch (RemediationCommitException e) { - rollbackRemediationWrites(instanceId, e.getRollbacks()); - throw new SkipRemediationException(SkipReason.SOURCE_WRITE_FAILED, e.getMessage(), e); - } - } catch (SkipRemediationException e) { - recordSkipped(skippedByReason, skipReasonLabel(e)); - LOG.warn("Skipping remediation {}: {}", instanceId, e.getMessage()); - LOG.debug("Skip reason for remediation {}: {}", instanceId, e.reason.displayName, e); - return false; - } catch (RollbackRemediationException e) { - throw e; - } catch (Exception e) { - recordSkipped(skippedByReason, SkipReason.UNEXPECTED_ERROR.displayName); - LOG.warn("Skipping remediation {} due to an unexpected processing error", instanceId); - LOG.debug("Unexpected error while processing remediation {}", instanceId, e); - return false; - } - } - - private Map prepareFileChanges(Element remediation, Path sourceBasePath, - FVDLMetadata fvdlMetadata) { - NodeList fileChangesNodes = remediation.getElementsByTagNameNS(NAMESPACE_URI, "FileChanges"); - if (fileChangesNodes.getLength() == 0) { - throw new SkipRemediationException(SkipReason.NO_CHANGES, "No file changes found"); - } - - Map pendingWrites = new LinkedHashMap<>(); - for (int j = 0; j < fileChangesNodes.getLength(); j++) { - processFileChanges(remediation, (Element) fileChangesNodes.item(j), sourceBasePath, fvdlMetadata, pendingWrites); - } - return pendingWrites; - } - - private boolean processFileChanges(Element remediation, Element fileChanges, Path sourceBasePath, FVDLMetadata fvdlMetadata, - Map pendingWrites) { - String instanceId = remediation.getAttribute("instanceId"); - String filename = getRequiredElementText(fileChanges, "Filename"); - Path filePath = sourceBasePath.resolve(filename).normalize(); - LOG.debug("Processing remediation {} file change for '{}' resolved to '{}'", instanceId, filename, filePath); - - if (!filePath.startsWith(sourceBasePath)) { - throw new SkipRemediationException(SkipReason.SOURCE_FILE_OUTSIDE_SOURCE_DIR, - "Source file resolves outside source directory: " + filename); - } - - if (!isFilePresent(filePath)) { - throw new SkipRemediationException(SkipReason.SOURCE_FILE_MISSING, "Source code file not present at: " + filePath); - } - - String fileHash = getRequiredElementText(fileChanges, "Hash"); - NodeList changesNodes = fileChanges.getElementsByTagNameNS(NAMESPACE_URI, "Change"); - if (changesNodes.getLength() == 0) { - throw new SkipRemediationException(SkipReason.NO_CHANGES, "No changes found for file: " + filename); - } - SourceFileContent sourceFileContent = getPendingOrSourceContent(filePath, filename, fvdlMetadata, pendingWrites); - Charset sourceEncoding = sourceFileContent.charset(); - LOG.debug("Remediation {} has {} change(s) for '{}' using source encoding {}", instanceId, changesNodes.getLength(), filename, - sourceFileContent.encodingSource()); - - String updatedContent = sourceFileContent.content(); - for (int k = 0; k < changesNodes.getLength(); k++) { - updatedContent = applyChange(instanceId, filename, fileHash, sourceEncoding, updatedContent, - (Element) changesNodes.item(k), k + 1); - } - byte[] updatedBytes = encodeSourceFile(updatedContent, sourceEncoding, filename); - pendingWrites.put(filePath, new PendingFileWrite(filename, filePath, updatedContent, sourceEncoding, - sourceFileContent.encodingSource(), updatedBytes)); - LOG.debug("Staged remediation {} for '{}' using source encoding {}; changes={}, encodedBytes={}", instanceId, filename, - sourceFileContent.encodingSource(), changesNodes.getLength(), updatedBytes.length); - return true; - } - - private String applyChange(String instanceId, String filename, String fileHash, Charset sourceEncoding, String originalContent, - Element change, int changeIndex) { - String lineSeparator = detectLineSeparator(originalContent); - String content = normalizeLineEndings(originalContent); - - List originalLines = Arrays.asList(content.split("\n", -1)); - LOG.debug("Decoded '{}' using {}; lineSeparator={}, normalizedLines={}", filename, sourceEncoding.name(), - describeLineSeparator(lineSeparator), originalLines.size()); - - int lineFrom = parseRequiredInt(change, "LineFrom"); - int lineTo = parseRequiredInt(change, "LineTo"); - LOG.debug("Remediation {} change {} for '{}' targets lines {}-{}", instanceId, changeIndex, filename, lineFrom, lineTo); - - String calculatedHash = calculateHashBase64(content, "SHA-256"); - boolean fileHashMatches = calculatedHash.equals(fileHash); - LOG.debug("Remediation {} hash check for '{}': {}", instanceId, filename, fileHashMatches ? "matched" : "mismatched"); - if (!fileHashMatches) { - LOG.debug("File hash mismatch for remediation {} in {}; searching changed source content", instanceId, filename); - Element contextElement = getRequiredElement(change, "Context"); - String contextText = contextElement.getTextContent(); - List contextLine = Arrays.asList(contextText.split("\\r?\\n")); - int contextLineFrom = fuzzySearchContext(instanceId, filename, originalLines, contextLine); - if (contextLineFrom == -1) { - LOG.debug("Context search failed for remediation {} in {}; context lines={}, source lines={}", instanceId, filename, - contextLine.size(), originalLines.size()); - throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_NOT_FOUND, "Source context not found for file '" + filename + - "'; file may have changed or remediation may overlap a previous change"); - } - LOG.debug("Context for remediation {} in {} matched at line {}", instanceId, filename, contextLineFrom + 1); - - String originalCodeText = getRequiredElementText(change, "OriginalCode"); - List originalCodeLine = Arrays.asList(originalCodeText.split("\\r?\\n")); - int contextBefore = parseRequiredContextAttribute(contextElement, "before"); - int contextAfter = parseRequiredContextAttribute(contextElement, "after"); - int[] lineFromTo = fuzzySearchOriginalCode(instanceId, filename, originalLines, originalCodeLine, - contextLineFrom, contextLine.size(), contextBefore, contextAfter); - if (lineFromTo[0] == -1 || lineFromTo[1] == -1) { - LOG.debug("Original code search failed for remediation {} in {}; context line={}, original code lines={}, source lines={}", - instanceId, filename, contextLineFrom + 1, originalCodeLine.size(), originalLines.size()); - throw new SkipRemediationException(SkipReason.ORIGINAL_CODE_NOT_FOUND, "Original code not found for file '" + filename + - "'; file may have changed or remediation may overlap a previous change"); - } - lineFrom = lineFromTo[0] + 1; - lineTo = lineFromTo[1] + 1; - LOG.debug("Original code for remediation {} in {} matched at lines {}-{}", instanceId, filename, lineFrom, lineTo); - } - - validateLineRange(lineFrom, lineTo, originalLines.size(), filename); - List newCodeLines = Arrays.asList(getRequiredElementText(change, "NewCode").split("\n")); - List updatedLines = new ArrayList<>(); - updatedLines.addAll(originalLines.subList(0, lineFrom - 1)); - updatedLines.addAll(newCodeLines); - updatedLines.addAll(originalLines.subList(lineTo, originalLines.size())); - LOG.debug("Staged remediation {} change {} for '{}' using FVDL encoding {}; updatedLines={}", instanceId, changeIndex, - filename, sourceEncoding.name(), updatedLines.size()); - return String.join(lineSeparator, updatedLines); - } - - private SourceFileContent getPendingOrSourceContent(Path filePath, String filename, FVDLMetadata fvdlMetadata, - Map pendingWrites) { - PendingFileWrite pendingWrite = pendingWrites.get(filePath); - return pendingWrite == null - ? readSourceFile(filePath, filename, fvdlMetadata) - : new SourceFileContent(pendingWrite.content(), pendingWrite.charset(), pendingWrite.encodingSource()); - } - - private void commitRemediationWrites(String instanceId, Map pendingWrites, Set modifiedFiles) - throws RemediationCommitException { - List rollbacks = new ArrayList<>(); - for (PendingFileWrite pendingWrite : pendingWrites.values()) { - try { - byte[] originalBytes = Files.readAllBytes(pendingWrite.filePath()); - rollbacks.add(new RollbackFileWrite(pendingWrite.filename(), pendingWrite.filePath(), originalBytes)); - LOG.debug("Writing remediation {} to '{}' using staged bytes; encodedBytes={}", instanceId, pendingWrite.filename(), - pendingWrite.updatedBytes().length); - Files.write(pendingWrite.filePath(), pendingWrite.updatedBytes()); - } catch (Exception e) { - throw new RemediationCommitException("Error writing source code file '" + pendingWrite.filename() + "'", e, rollbacks); - } - } - - for (PendingFileWrite pendingWrite : pendingWrites.values()) { - modifiedFiles.add(pendingWrite.filename()); - LOG.info("Remediation applied for {} in file {}", instanceId, pendingWrite.filename()); - } - } - - private void rollbackRemediationWrites(String instanceId, List rollbacks) { - for (RollbackFileWrite rollback : rollbacks) { - try { - Files.write(rollback.filePath(), rollback.originalBytes()); - LOG.warn("Rolled back remediation {} changes for '{}' after write failure", instanceId, rollback.filename()); - } catch (IOException rollbackException) { - LOG.error("Failed to roll back remediation {} changes for '{}'", instanceId, rollback.filename(), rollbackException); - throw new RollbackRemediationException("Failed to roll back remediation changes for '" + rollback.filename() + - "'. Source files may be partially modified; inspect the source tree before retrying", rollbackException); - } - } - } - - private int fuzzySearchContext(String instanceId, String filename, List originalLines, List contextLine) { - try { - List matches = FuzzyContextSearcher.fuzzySearchContextMatches(originalLines, contextLine, 0); - if (matches.size() > 1) { - String candidateLines = matches.stream() - .map(line -> String.valueOf(line + 1)) - .collect(Collectors.joining(", ")); - throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_AMBIGUOUS, - "Source context matched multiple locations in file '" + filename + "'; candidate lines: " + candidateLines); - } - return matches.isEmpty() ? -1 : matches.get(0); - } catch (IOException e) { - throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_NOT_FOUND, - "Error searching source context for remediation '" + instanceId + "' in file '" + filename + "'", e); - } - } - - private int[] fuzzySearchOriginalCode(String instanceId, String filename, List originalLines, List originalCodeLine, - int contextLineFrom, int contextLineCount, int contextBefore, int contextAfter) { - int contextStart = contextLineFrom + contextBefore; - int contextEnd = contextLineFrom + contextLineCount - contextAfter; - if (contextStart < 0 || contextStart >= contextEnd || contextEnd > originalLines.size()) { - return new int[] {-1, -1}; - } - - int[] lineFromTo = FuzzyContextSearcher.fuzzySearchOriginalCode( - originalLines.subList(contextStart, contextEnd), originalCodeLine, 0, 0); - if (lineFromTo[0] == -1 || lineFromTo[1] == -1) { - return lineFromTo; - } - return new int[] {lineFromTo[0] + contextStart, lineFromTo[1] + contextStart}; - } - - private boolean isFilePresent(Path path) { - return Files.exists(path) && Files.isRegularFile(path); - } - - /** Nullable: missing/unreadable FVDL means FPR encoding candidate is skipped. */ - private FVDLMetadata loadFvdlMetadata() { - if (!Files.exists(fprHandle.getPath("/audit.fvdl"))) { - LOG.warn("FVDL file '/audit.fvdl' is missing; FPR encoding candidate will be skipped"); - return null; - } - - try (ZipFile zipFile = new ZipFile(fprHandle.getFprPath().toFile())) { - LOG.debug("Loading FVDL build metadata from '{}' to resolve source encodings", fprHandle.getFprPath()); - // Decoder unused for metadata-only parse; ctor requires one for FileUtils wiring. - StreamingFVDLProcessor processor = new StreamingFVDLProcessor(fprHandle, sourceDecoder); - processor.parseBuildMetadata(zipFile, "audit.fvdl"); - LOG.debug("Loaded FVDL build metadata from '{}'", fprHandle.getFprPath()); - return processor.getFvdlMetadata(); - } catch (Exception e) { - LOG.warn("Error reading source file encodings from audit.fvdl; FPR encoding candidate will be skipped", e); - return null; - } - } - - private SourceFileContent readSourceFile(Path filePath, String filename, FVDLMetadata fvdlMetadata) { - try { - byte[] sourceBytes = Files.readAllBytes(filePath); - // Metadata may be null (FVDL missing); FPR candidate fails and other encodings are tried. - DecodeResult decodeResult = sourceDecoder.decode(sourceBytes, filename, fvdlMetadata); - LOG.debug("Strict decoded '{}' using {}; sourceBytes={}, decodedChars={}", filename, decodeResult.source(), sourceBytes.length, - decodeResult.content().length()); - return new SourceFileContent(decodeResult.content(), decodeResult.charset(), decodeResult.source()); - } catch (SourceDecodeException e) { - throw new SkipRemediationException(SkipReason.SOURCE_DECODE_FAILED, e.getMessage(), e); - } catch (IOException e) { - throw new SkipRemediationException(SkipReason.SOURCE_READ_FAILED, "Error reading source code file '" + filePath + "'", e); - } - } - - private byte[] encodeSourceFile(String content, Charset charset, String filename) { - try { - return SourceEncoder.encode(content, charset, filename); - } catch (SourceEncodeException e) { - throw new SkipRemediationException(SkipReason.REMEDIATION_ENCODE_FAILED, e.getMessage(), e); - } - } - - private String getRequiredElementText(Element parent, String elementName) { - return getRequiredElement(parent, elementName).getTextContent(); - } - - private Element getRequiredElement(Element parent, String elementName) { - NodeList nodes = parent.getElementsByTagNameNS(NAMESPACE_URI, elementName); - if (nodes.getLength() == 0 || nodes.item(0) == null) { - throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, - "Missing required remediation element '" + elementName + "'"); - } - return (Element) nodes.item(0); - } - - private int parseRequiredContextAttribute(Element context, String attributeName) { - String value = context.getAttribute(attributeName); - if (value == null || value.isBlank()) { - throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, - "Missing required remediation context attribute '" + attributeName + "'"); - } - try { - int parsedValue = Integer.parseInt(value); - if (parsedValue < 0) { - throw new NumberFormatException("negative value"); - } - return parsedValue; - } catch (NumberFormatException e) { - throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, - "Invalid remediation context attribute '" + attributeName + "': " + value, e); - } - } - - private int parseRequiredInt(Element parent, String elementName) { - String value = getRequiredElementText(parent, elementName); - try { - return Integer.parseInt(value); - } catch (NumberFormatException e) { - throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, - "Invalid integer value for remediation element '" + elementName + "': " + value, e); - } - } - - private void validateLineRange(int lineFrom, int lineTo, int sourceLineCount, String filename) { - if (lineFrom < 1 || lineTo < lineFrom || lineTo > sourceLineCount) { - throw new SkipRemediationException(SkipReason.REMEDIATION_LINE_RANGE_INVALID, - "Invalid remediation line range " + lineFrom + "-" + lineTo + " for file '" + filename + "'"); - } - } - - private String detectLineSeparator(String content) { - int crlfIndex = content.indexOf("\r\n"); - int lfIndex = content.indexOf('\n'); - int crIndex = content.indexOf('\r'); - - if (crlfIndex >= 0 && (lfIndex == crlfIndex + 1 || lfIndex < 0) && (crIndex == crlfIndex || crIndex < 0)) { - return "\r\n"; - } - if (lfIndex >= 0 && (crIndex < 0 || lfIndex < crIndex)) { - return "\n"; - } - if (crIndex >= 0) { - return "\r"; - } - return System.lineSeparator(); - } - - private String normalizeLineEndings(String content) { - return content.replace("\r\n", "\n").replace('\r', '\n'); - } - - private String describeLineSeparator(String lineSeparator) { - return switch (lineSeparator) { - case "\r\n" -> "CRLF"; - case "\n" -> "LF"; - case "\r" -> "CR"; - default -> "system"; - }; - } - - private String calculateHashBase64(String content, String algorithm) { - String hash; - if (content == null) { - return ""; - } - try { - MessageDigest md = MessageDigest.getInstance(algorithm); - byte[] digest = md.digest(content.getBytes(StandardCharsets.UTF_8)); - hash = Base64.getEncoder().encodeToString(digest); - return hash; - } catch (NoSuchAlgorithmException e) { - throw new AviatorTechnicalException("Hashing algorithm not available: " + algorithm, e); - } - } - - private void recordSkipped(Map skippedByReason, String reason) { - skippedByReason.merge(reason, 1, Integer::sum); - } - - private String skipReasonLabel(SkipRemediationException exception) { - return exception.reason.displayName; - } - - private String formatSkippedReasons(Map skippedByReason) { - List parts = new ArrayList<>(); - skippedByReason.forEach((reason, count) -> parts.add(reason + "=" + count)); - return String.join(", ", parts); - } -} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/RemediationProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/RemediationProcessor.java new file mode 100644 index 00000000000..24f1e9ee430 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/RemediationProcessor.java @@ -0,0 +1,334 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.zip.ZipFile; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; + +import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.processor.StreamingFVDLProcessor; +import com.fortify.cli.aviator.fpr.remediation.applier.RemediationApplier; +import com.fortify.cli.aviator.fpr.remediation.classifier.AppliedChangeLedger; +import com.fortify.cli.aviator.fpr.remediation.classifier.HunkClassifier; +import com.fortify.cli.aviator.fpr.remediation.exception.RemediationCommitException; +import com.fortify.cli.aviator.fpr.remediation.exception.RollbackRemediationException; +import com.fortify.cli.aviator.fpr.remediation.exception.SkipRemediationException; +import com.fortify.cli.aviator.fpr.remediation.model.HunkOutcome; +import com.fortify.cli.aviator.fpr.remediation.model.Remediation; +import com.fortify.cli.aviator.fpr.remediation.model.RemediationDocument; +import com.fortify.cli.aviator.fpr.remediation.model.RemediationKey; +import com.fortify.cli.aviator.fpr.remediation.model.RemediationMetric; +import com.fortify.cli.aviator.fpr.remediation.writer.FileWriteCoordinator; +import com.fortify.cli.aviator.fpr.remediation.writer.PendingFileWrite; +import com.fortify.cli.aviator.fpr.remediation.writer.PreparedFileChanges; +import com.fortify.cli.aviator.fpr.remediation.xmlprocessor.RemediationDocumentMapper; +import com.fortify.cli.aviator.fpr.remediation.xmlprocessor.RemediationXmlReader; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; +import com.fortify.cli.aviator.util.FprHandle; + +/** + * Orchestrator. {@link #processRemediationXML()} runs the three phases in sequence: parse + * XML into a DOM {@link Document} ({@link RemediationXmlReader}), map the document into the + * domain model with zero business logic ({@link RemediationDocumentMapper}), then classify + * and apply each remediation ({@link #classifyAndApply}). Public API (constructors, method + * signature) is unchanged from the original single-class implementation. + * + *

Per-FPR isolation: Each processor instance handles exactly one FPR. In multi-FPR + * scenarios (e.g., --all-open-issues), each artifact gets its own processor with a fresh + * {@link AppliedChangeLedger}. Ledgers are NOT shared across artifacts because line numbers + * are relative to pristine-file coordinates, and different artifacts are scans of potentially + * different source revisions. Correctness across FPRs is maintained by anchor verification + * (hash checking): once an earlier FPR has touched a file, the declared hash no longer matches, + * so later hunks apply only where their OriginalCode still literally matches. + */ +public class RemediationProcessor { + private static final Logger LOG = LoggerFactory.getLogger(RemediationProcessor.class); + + private final FprHandle fprHandle; + private final String sourceCodeDirectory; + private final ISourceDecoder sourceDecoder; + + private final RemediationXmlReader xmlReader = new RemediationXmlReader(); + private final RemediationDocumentMapper documentMapper = new RemediationDocumentMapper(); + private final HunkClassifier hunkClassifier = new HunkClassifier(); + private final RemediationApplier remediationApplier = new RemediationApplier(); + private final FileWriteCoordinator fileWriteCoordinator; + + public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory) { + this(fprHandle, sourceCodeDirectory, SourceDecoders.defaults()); + } + + public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory, ISourceDecoder sourceDecoder) { + this.fprHandle = fprHandle; + this.sourceCodeDirectory = sourceCodeDirectory; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); + this.fileWriteCoordinator = new FileWriteCoordinator(sourceDecoder, remediationApplier); + } + + public RemediationMetric processRemediationXML() { + Path remediationPath = fprHandle.getPath("/remediations.xml"); + Path sourceBasePath = resolveSourceBasePath(); + LOG.debug("Applying remediations from {} to source directory {}", remediationPath, sourceBasePath); + FVDLMetadata fvdlMetadata = loadFvdlMetadata(); + + try { + Document remediationDoc = xmlReader.read(remediationPath); + RemediationDocument remediations = documentMapper.map(remediationDoc); + AppliedChangeLedger ledger = new AppliedChangeLedger(); + return classifyAndApply(remediations, sourceBasePath, fvdlMetadata, ledger); + } catch (AviatorTechnicalException e) { + throw e; + } catch (Exception e) { + LOG.error("Unexpected error processing remediation.xml: {}", remediationPath, e); + throw new AviatorTechnicalException("Unexpected error processing remediations.xml.", e); + } + } + + private Path resolveSourceBasePath() { + String trimmedSourceDir = sourceCodeDirectory.trim(); + if (trimmedSourceDir.length() > 1 && + ((trimmedSourceDir.startsWith("\"") && trimmedSourceDir.endsWith("\"")) || + (trimmedSourceDir.startsWith("'") && trimmedSourceDir.endsWith("'")))) { + trimmedSourceDir = trimmedSourceDir.substring(1, trimmedSourceDir.length() - 1); + } + return Paths.get(trimmedSourceDir).toAbsolutePath().normalize(); + } + + private RemediationMetric classifyAndApply(RemediationDocument remediationDocument, Path sourceBasePath, FVDLMetadata fvdlMetadata, + AppliedChangeLedger ledger) { + List orderedRemediations = new ArrayList<>(remediationDocument.remediations()); + int totalRemediations = orderedRemediations.size(); + LOG.debug("Loaded {} remediation entries", totalRemediations); + try { + // Widest-first ordering: broader fixes land first so narrower nested ones classify as SUPERSEDED. + orderedRemediations.sort((a, b) -> Integer.compare(b.maxHunkWidth(), a.maxHunkWidth())); + } catch (SkipRemediationException e) { + LOG.debug("Unable to sort remediations by width; proceeding with original order", e); + } + int appliedRemediations = 0; + int identicalRemediations = 0; + int supersededRemediations = 0; + int possiblyRemediatedRemediations = 0; + Set modifiedFiles = new LinkedHashSet<>(); + Map skippedByReason = new LinkedHashMap<>(); + Map remediationLookup = new LinkedHashMap<>(); + + for (Remediation remediation : orderedRemediations) { + String instanceId = remediation.instanceId(); + List remediationKeys; + try { + remediationKeys = remediation.createRemediationKeys(sourceBasePath); + } catch (SkipRemediationException e) { + recordSkipped(skippedByReason, skipReasonLabel(e)); + LOG.warn("Skipping remediation {}: {}", instanceId, e.getMessage()); + LOG.debug("Skip reason for remediation {}: {}", instanceId, e.getReason().displayName(), e); + continue; + } + + // Hunk-level identity: partition keys into already-satisfied vs to-apply. + Set satisfiedKeys = new LinkedHashSet<>(); + Set toApplyKeys = new LinkedHashSet<>(); + Set satisfiedByInstances = new LinkedHashSet<>(); + for (RemediationKey key : remediationKeys) { + String owner = remediationLookup.get(key); + if (owner != null) { + satisfiedKeys.add(key); + satisfiedByInstances.add(owner); + } else { + toApplyKeys.add(key); + } + } + + // Fully identical: every hunk was already applied by an earlier remediation with same content. + if (!remediationKeys.isEmpty() && toApplyKeys.isEmpty()) { + identicalRemediations++; + LOG.info("Remediation {} is fully identical to prior remediation(s) {}; {} hunk(s) already applied", + instanceId, satisfiedByInstances, satisfiedKeys.size()); + continue; + } + + // SUPERSEDED / CONFLICTS pre-check: classify each unsatisfied hunk against the ledger. + List preClass = hunkClassifier.classifyRemediationHunks(remediation, sourceBasePath, ledger); + boolean anyApplyCandidate = preClass.stream().anyMatch(o -> o == HunkOutcome.APPLIED); + boolean allSuperseded = !preClass.isEmpty() && preClass.stream().allMatch(o -> o == HunkOutcome.SUPERSEDED); + boolean anyConflicts = preClass.stream().anyMatch(o -> o == HunkOutcome.CONFLICTS); + boolean allPossiblyRemediated = !preClass.isEmpty() + && preClass.stream().noneMatch(o -> o == HunkOutcome.APPLIED || o == HunkOutcome.CONFLICTS) + && preClass.stream().anyMatch(o -> o == HunkOutcome.POSSIBLY_REMEDIATED); + + if (!anyApplyCandidate && allSuperseded) { + supersededRemediations++; + LOG.info("Remediation {} is superseded by a broader prior fix for all {} hunk(s); no write needed", + instanceId, preClass.size()); + continue; + } + // Remediations are applied atomically: even one CONFLICTS hunk means this remediation + // cannot be fully/correctly applied, so reject it now rather than let the applier's + // best-effort offset/fuzzy-anchor recovery (meant for non-conflicting shifts) decide. + if (anyConflicts) { + recordSkipped(skippedByReason, SkipReason.CONFLICTS_WITH_ANOTHER_FIX.displayName()); + LOG.info("Remediation {} conflicts with prior fix(es) on {} of {} hunk(s); skipping", + instanceId, preClass.stream().filter(o -> o == HunkOutcome.CONFLICTS).count(), preClass.size()); + continue; + } + if (!anyApplyCandidate && allPossiblyRemediated) { + possiblyRemediatedRemediations++; + LOG.info("Remediation {} possibly remediated by a sibling fix with different content for all {} hunk(s)", + instanceId, preClass.size()); + continue; + } + + // Partial identity: some hunks already applied; apply only the rest. + if (!satisfiedKeys.isEmpty()) { + LOG.info("Remediation {} is partially identical to prior remediation(s) {}; {} of {} hunk(s) already applied, {} still to apply", + instanceId, satisfiedByInstances, satisfiedKeys.size(), remediationKeys.size(), toApplyKeys.size()); + } + + // Mixed classification: a hunk already covered by a broader prior fix (SUPERSEDED or + // POSSIBLY_REMEDIATED) must not be re-attempted alongside a genuinely-applicable + // sibling hunk, or the covered hunk's stale anchor drags the whole remediation down. + // preClass and remediationKeys iterate the same fileChanges/hunks in the same order. + boolean classifierNarrowed = false; + for (int i = 0; i < preClass.size(); i++) { + HunkOutcome outcome = preClass.get(i); + if (outcome == HunkOutcome.SUPERSEDED || outcome == HunkOutcome.POSSIBLY_REMEDIATED) { + if (toApplyKeys.remove(remediationKeys.get(i))) { + classifierNarrowed = true; + } + } + } + if (classifierNarrowed) { + LOG.info("Remediation {} has {} hunk(s) already covered by a broader prior fix; applying only the rest", + instanceId, remediationKeys.size() - toApplyKeys.size() - satisfiedKeys.size()); + } + + Set filter = (satisfiedKeys.isEmpty() && !classifierNarrowed) ? null : toApplyKeys; + Set applied = processRemediation(remediation, sourceBasePath, fvdlMetadata, modifiedFiles, skippedByReason, filter, ledger); + if (!applied.isEmpty()) { + appliedRemediations++; + for (RemediationKey key : applied) { + LOG.debug("putting {}", instanceId); + remediationLookup.put(key, instanceId); + } + } + } + + int skippedRemediations = totalRemediations - appliedRemediations - identicalRemediations - supersededRemediations + - possiblyRemediatedRemediations; + LOG.info("Auto-remediation summary: total={}, applied={}, identical={}, superseded={}, possiblyRemediated={}, skipped={}", + totalRemediations, appliedRemediations, identicalRemediations, supersededRemediations, possiblyRemediatedRemediations, + skippedRemediations); + if (!skippedByReason.isEmpty()) { + LOG.info("Skipped remediations by reason: {}", formatSkippedReasons(skippedByReason)); + } + return RemediationMetric.builder() + .totalRemediations(totalRemediations) + .appliedRemediations(appliedRemediations) + .identicalRemediations(identicalRemediations) + .supersededRemediations(supersededRemediations) + .possiblyRemediatedRemediations(possiblyRemediatedRemediations) + .skippedRemediations(skippedRemediations) + .modifiedFiles(modifiedFiles) + .skippedByReason(skippedByReason) + .build(); + } + + private Set processRemediation(Remediation remediation, Path sourceBasePath, FVDLMetadata fvdlMetadata, + Set modifiedFiles, Map skippedByReason, Set keysToApply, + AppliedChangeLedger ledger) { + String instanceId = remediation.instanceId(); + ledger.discardStaged(); + try { + PreparedFileChanges prepared = fileWriteCoordinator.prepareFileChanges(remediation, sourceBasePath, fvdlMetadata, keysToApply, ledger); + Map pendingWrites = prepared.pendingWrites(); + + if (pendingWrites.isEmpty()) { + recordSkipped(skippedByReason, SkipReason.NO_CHANGES.displayName()); + return Set.of(); + } + try { + fileWriteCoordinator.commitRemediationWrites(instanceId, pendingWrites, modifiedFiles); + // Only on successful commit do the staged hunks enter the per-run offset map. + ledger.commitStaged(); + return prepared.appliedKeys(); + } catch (RemediationCommitException e) { + ledger.discardStaged(); + fileWriteCoordinator.rollbackRemediationWrites(instanceId, e.getRollbacks()); + throw new SkipRemediationException(SkipReason.SOURCE_WRITE_FAILED, e.getMessage(), e); + } + } catch (SkipRemediationException e) { + ledger.discardStaged(); + recordSkipped(skippedByReason, skipReasonLabel(e)); + LOG.warn("Skipping remediation {}: {}", instanceId, e.getMessage()); + LOG.debug("Skip reason for remediation {}: {}", instanceId, e.getReason().displayName(), e); + return Set.of(); + } catch (RollbackRemediationException e) { + throw e; + } catch (Exception e) { + ledger.discardStaged(); + recordSkipped(skippedByReason, SkipReason.UNEXPECTED_ERROR.displayName()); + LOG.warn("Skipping remediation {} due to an unexpected processing error", instanceId); + LOG.debug("Unexpected error while processing remediation {}", instanceId, e); + return Set.of(); + } + } + + private void recordSkipped(Map skippedByReason, String reason) { + skippedByReason.merge(reason, 1, Integer::sum); + } + + private String skipReasonLabel(SkipRemediationException exception) { + return exception.getReason().displayName(); + } + + private String formatSkippedReasons(Map skippedByReason) { + List parts = new ArrayList<>(); + skippedByReason.forEach((reason, count) -> parts.add(reason + "=" + count)); + return String.join(", ", parts); + } + + /** Nullable: missing/unreadable FVDL means FPR encoding candidate is skipped. */ + private FVDLMetadata loadFvdlMetadata() { + if (!Files.exists(fprHandle.getPath("/audit.fvdl"))) { + LOG.warn("FVDL file '/audit.fvdl' is missing; FPR encoding candidate will be skipped"); + return null; + } + + try (ZipFile zipFile = new ZipFile(fprHandle.getFprPath().toFile())) { + LOG.debug("Loading FVDL build metadata from '{}' to resolve source encodings", fprHandle.getFprPath()); + // Decoder unused for metadata-only parse; ctor requires one for FileUtils wiring. + StreamingFVDLProcessor processor = new StreamingFVDLProcessor(fprHandle, sourceDecoder); + processor.parseBuildMetadata(zipFile, "audit.fvdl"); + LOG.debug("Loaded FVDL build metadata from '{}'", fprHandle.getFprPath()); + return processor.getFvdlMetadata(); + } catch (Exception e) { + LOG.warn("Error reading source file encodings from audit.fvdl; FPR encoding candidate will be skipped", e); + return null; + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/SkipReason.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/SkipReason.java new file mode 100644 index 00000000000..563ed813be4 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/SkipReason.java @@ -0,0 +1,42 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation; + + public enum SkipReason { + SOURCE_FILE_MISSING("Source file missing"), + SOURCE_FILE_OUTSIDE_SOURCE_DIR("Source file outside source directory"), + SOURCE_READ_FAILED("Source file read failed"), + SOURCE_DECODE_FAILED("Source file decode failed"), + REMEDIATION_DATA_INVALID("Remediation data invalid"), + REMEDIATION_LINE_RANGE_INVALID("Remediation line range invalid"), + SOURCE_CONTEXT_NOT_FOUND("Source context not found"), + SOURCE_CONTEXT_AMBIGUOUS("Source context matched multiple locations"), + ORIGINAL_CODE_NOT_FOUND("Original code not found"), + ORIGINAL_CODE_AMBIGUOUS("Original code matched multiple locations"), + CONFLICTS_WITH_ANOTHER_FIX("Conflicts with another fix"), + ANCHOR_DOES_NOT_MATCH("Anchor does not match"), + REMEDIATION_ENCODE_FAILED("Remediation encode failed"), + SOURCE_WRITE_FAILED("Source file write failed"), + NO_CHANGES("No file changes found"), + UNEXPECTED_ERROR("Unexpected remediation processing error"); + + final String displayName; + + SkipReason(String displayName) { + this.displayName = displayName; + } + + final String displayName() { + return displayName; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/AppliedChangeResult.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/AppliedChangeResult.java new file mode 100644 index 00000000000..98753e08f6b --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/AppliedChangeResult.java @@ -0,0 +1,15 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.applier; + +public record AppliedChangeResult(String updatedContent, int actualLineFrom, int actualLineTo) {} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/FuzzyAnchorLocator.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/FuzzyAnchorLocator.java new file mode 100644 index 00000000000..54010d285e5 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/FuzzyAnchorLocator.java @@ -0,0 +1,81 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.applier; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.fortify.cli.aviator.fpr.remediation.SkipReason; +import com.fortify.cli.aviator.fpr.remediation.exception.SkipRemediationException; +import com.fortify.cli.aviator.util.FuzzyContextSearcher; + +/** Wraps the FuzzyContextSearcher utility's context/original-code matching, unmodified from the original. */ +public final class FuzzyAnchorLocator { + + public int searchContext(String instanceId, String filename, List originalLines, List contextLine, + int projectedDeclaredFrom, int contextBefore) { + try { + List matches = FuzzyContextSearcher.fuzzySearchContextMatches(originalLines, contextLine, 0); + if (matches.size() > 1) { + int expectedContextLineFrom = projectedDeclaredFrom - 1 - contextBefore; + List exact = matches.stream().filter(m -> m == expectedContextLineFrom).toList(); + if (exact.size() == 1) { + return exact.get(0); + } + String candidateLines = matches.stream() + .map(line -> String.valueOf(line + 1)) + .collect(Collectors.joining(", ")); + throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_AMBIGUOUS, + "Source context matched multiple locations in file '" + filename + "'; candidate lines: " + candidateLines); + } + return matches.isEmpty() ? -1 : matches.get(0); + } catch (IOException e) { + throw new SkipRemediationException(SkipReason.SOURCE_CONTEXT_NOT_FOUND, + "Error searching source context for remediation '" + instanceId + "' in file '" + filename + "'", e); + } + } + + public Optional searchOriginalCode(String instanceId, String filename, List originalLines, List originalCodeLine, + int contextLineFrom, int contextLineCount, int contextBefore, int contextAfter, + int projectedDeclaredFrom, int projectedDeclaredTo) { + int contextStart = contextLineFrom + contextBefore; + int contextEnd = contextLineFrom + contextLineCount - contextAfter; + if (contextStart < 0 || contextStart >= contextEnd || contextEnd > originalLines.size()) { + return Optional.empty(); + } + + List matches = FuzzyContextSearcher.fuzzySearchOriginalCodeMatches( + originalLines.subList(contextStart, contextEnd), originalCodeLine, 0, 0); + if (matches.size() > 1) { + int expectedFrom = projectedDeclaredFrom - 1 - contextStart; + int expectedTo = projectedDeclaredTo - 1 - contextStart; + List exact = matches.stream().filter(m -> m.from() == expectedFrom && m.to() == expectedTo).toList(); + if (exact.size() == 1) { + LineRange m = exact.get(0); + return Optional.of(new LineRange(m.from() + contextStart, m.to() + contextStart)); + } + String candidateLines = matches.stream() + .map(m -> String.valueOf(m.from() + contextStart + 1)) + .collect(Collectors.joining(", ")); + throw new SkipRemediationException(SkipReason.ORIGINAL_CODE_AMBIGUOUS, + "Original code matched multiple locations in file '" + filename + "'; candidate lines: " + candidateLines); + } + if (matches.isEmpty()) { + return Optional.empty(); + } + LineRange lineFromTo = matches.get(0); + return Optional.of(new LineRange(lineFromTo.from() + contextStart, lineFromTo.to() + contextStart)); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/LineRange.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/LineRange.java new file mode 100644 index 00000000000..5a75c69f77c --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/LineRange.java @@ -0,0 +1,17 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.applier; + +/** Inclusive 0-based [from, to] line range located by fuzzy anchor search. */ +public record LineRange(int from, int to) { +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/RemediationApplier.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/RemediationApplier.java new file mode 100644 index 00000000000..71655ebef77 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/applier/RemediationApplier.java @@ -0,0 +1,308 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.applier; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; +import com.fortify.cli.aviator.fpr.remediation.SkipReason; +import com.fortify.cli.aviator.fpr.remediation.classifier.AppliedChangeLedger; +import com.fortify.cli.aviator.fpr.remediation.exception.SkipRemediationException; +import com.fortify.cli.aviator.fpr.remediation.model.AppliedChange; +import com.fortify.cli.aviator.fpr.remediation.model.Hunk; +import com.fortify.cli.aviator.util.FileUtil; + +/** Houses the original {@code applyChange}, split into the same steps it always tried, just named. */ +public final class RemediationApplier { + private static final Logger LOG = LoggerFactory.getLogger(RemediationApplier.class); + + private final FuzzyAnchorLocator fuzzyAnchorLocator = new FuzzyAnchorLocator(); + + public AppliedChangeResult applyChange(String instanceId, String filename, Path filePath, String fileHash, Charset sourceEncoding, + String originalContent, Hunk hunk, int changeIndex, AppliedChangeLedger ledger) { + String lineSeparator = detectLineSeparator(originalContent); + String content = normalizeLineEndings(originalContent); + + List originalLines = Arrays.asList(content.split("\n", -1)); + LOG.debug("Decoded '{}' using {}; lineSeparator={}, normalizedLines={}", filename, sourceEncoding.name(), + describeLineSeparator(lineSeparator), originalLines.size()); + + int lineFrom = hunk.lineFrom(); + int lineTo = hunk.lineTo(); + LOG.debug("Remediation {} change {} for '{}' targets lines {}-{}", instanceId, changeIndex, filename, lineFrom, lineTo); + + boolean fileHashMatches = tryHashMatch(instanceId, filename, fileHash, sourceEncoding, content, originalContent); + if (!fileHashMatches) { + LOG.debug("File hash mismatch for remediation {} in {}; searching changed source content", instanceId, filename); + List priorApplied = ledger.changesFor(filePath); + + Optional projected = tryOffsetProjection(instanceId, filename, hunk, originalLines, lineFrom, lineTo, priorApplied, ledger, filePath); + if (projected.isPresent()) { + lineFrom = projected.get().from(); + lineTo = projected.get().to(); + } else { + LineRange anchored = tryFuzzyAnchor(instanceId, filename, hunk, originalLines, priorApplied, + lineFrom, lineTo, filePath, ledger); + lineFrom = anchored.from(); + lineTo = anchored.to(); + } + } + + validateLineRange(lineFrom, lineTo, originalLines.size(), filename); + // Normalize line endings the same way the file content already was (line 46): unmapped + // extensions get no such normalization from stripSyntheticLineMarkers, so a CRLF NewCode + // would otherwise leave a stray CR on every line once rejoined with the file's own separator. + String normalizedNewCode = normalizeLineEndings(FileUtil.stripSyntheticLineMarkers(hunk.requiredNewCode(), filename)); + List newCodeLines = new ArrayList<>(Arrays.asList(normalizedNewCode.split("\n"))); + dropDuplicatedBoundaryTokens(newCodeLines, originalLines, lineFrom, lineTo, instanceId, filename); + List updatedLines = new ArrayList<>(); + updatedLines.addAll(originalLines.subList(0, lineFrom - 1)); + updatedLines.addAll(newCodeLines); + updatedLines.addAll(originalLines.subList(lineTo, originalLines.size())); + LOG.debug("Staged remediation {} change {} for '{}' using FVDL encoding {}; updatedLines={}", instanceId, changeIndex, + filename, sourceEncoding.name(), updatedLines.size()); + String updatedContent = String.join(lineSeparator, updatedLines); + return new AppliedChangeResult(updatedContent, lineFrom, lineTo); + } + + /** + * Try canonical hash first (matches the new AuditProcessor form), then legacy raw-content + * hash so pre-fix FPRs still match. Try each with BOTH UTF-8 and the file's declared source + * encoding — the doc's "5 of 53 files not valid UTF-8" case fails when audit and apply disagree + * on the encoding used for the getBytes step; accepting the source-encoding form covers it. + */ + private boolean tryHashMatch(String instanceId, String filename, String fileHash, Charset sourceEncoding, String content, + String originalContent) { + String canonicalStr = FileUtil.canonicalizeForHash(content); + String legacyStr = originalContent; + String canonicalHashUtf8 = calculateHashBase64Bytes(canonicalStr.getBytes(StandardCharsets.UTF_8), "SHA-256"); + String legacyHashUtf8 = calculateHashBase64Bytes(legacyStr.getBytes(StandardCharsets.UTF_8), "SHA-256"); + String canonicalHashSrc = calculateHashBase64Bytes(canonicalStr.getBytes(sourceEncoding), "SHA-256"); + String legacyHashSrc = calculateHashBase64Bytes(legacyStr.getBytes(sourceEncoding), "SHA-256"); + boolean fileHashMatches; + String matchedForm; + if (canonicalHashUtf8.equals(fileHash)) { + fileHashMatches = true; + matchedForm = "canonical"; + } else if (legacyHashUtf8.equals(fileHash)) { + fileHashMatches = true; + matchedForm = "legacy"; + } else if (canonicalHashSrc.equals(fileHash)) { + fileHashMatches = true; + matchedForm = "canonical/" + sourceEncoding.name(); + } else if (legacyHashSrc.equals(fileHash)) { + fileHashMatches = true; + matchedForm = "legacy/" + sourceEncoding.name(); + } else { + fileHashMatches = false; + matchedForm = "none"; + } + LOG.debug("Remediation {} hash check for '{}': {}", + instanceId, filename, fileHashMatches ? ("matched (" + matchedForm + ")") : "mismatched"); + return fileHashMatches; + } + + /** + * If a prior remediation this run modified this file, project the declared range through + * the accumulated line-delta of every AppliedChange whose original range sits strictly + * before this hunk's declared start. Verify the projected position holds the expected + * OriginalCode (whitespace-insensitive). Returns {@code null} (try the fuzzy fallback + * instead) if there is no prior history, the projected range is out of bounds, or the + * anchor at the projected position doesn't match. + */ + private Optional tryOffsetProjection(String instanceId, String filename, Hunk hunk, List originalLines, + int lineFrom, int lineTo, List priorApplied, AppliedChangeLedger ledger, Path filePath) { + if (priorApplied.isEmpty()) { + return Optional.empty(); + } + int shift = ledger.projectOffset(filePath, lineFrom); + int projectedFrom = lineFrom + shift; + int projectedTo = lineTo + shift; + if (projectedFrom >= 1 && projectedTo >= projectedFrom && projectedTo <= originalLines.size()) { + String originalCodeText = hunk.requiredOriginalCode(); + List originalCodeLines = Arrays.asList(originalCodeText.split("\\r?\\n")); + if (linesEqualNormalized(originalLines, projectedFrom - 1, projectedTo - 1, originalCodeLines)) { + LOG.debug("Remediation {} projected via offset map for '{}': declared {}-{} shifted by {} to {}-{}", + instanceId, filename, lineFrom, lineTo, shift, projectedFrom, projectedTo); + return Optional.of(new LineRange(projectedFrom, projectedTo)); + } else { + LOG.debug("Remediation {} projection anchor mismatch for '{}' at projected {}-{}; falling back", + instanceId, filename, projectedFrom, projectedTo); + } + } + return Optional.empty(); + } + + /** Context search first, then a whole-file OriginalCode fallback if no context match was found. */ + private LineRange tryFuzzyAnchor(String instanceId, String filename, Hunk hunk, List originalLines, + List priorApplied, int lineFrom, int lineTo, Path filePath, AppliedChangeLedger ledger) { + int shift = ledger.projectOffset(filePath, lineFrom); + int projectedFrom = lineFrom + shift; + int projectedTo = lineTo + shift; + String contextText = hunk.requiredContextText(); + List contextLine = Arrays.asList(contextText.split("\\r?\\n")); + int contextBefore = hunk.contextBefore(); + int contextAfter = hunk.contextAfter(); + int contextLineFrom = fuzzyAnchorLocator.searchContext(instanceId, filename, originalLines, contextLine, + projectedFrom, contextBefore); + if (contextLineFrom == -1) { + LOG.debug("Context search failed for remediation {} in {}; trying whole-file OriginalCode fallback", + instanceId, filename); + String fallbackOriginalCodeText = hunk.requiredOriginalCode(); + List fallbackOriginalCodeLine = Arrays.asList(fallbackOriginalCodeText.split("\\r?\\n")); + Optional wholeFile = fuzzyAnchorLocator.searchOriginalCode(instanceId, filename, originalLines, fallbackOriginalCodeLine, + 0, originalLines.size(), 0, 0, projectedFrom, projectedTo); + if (wholeFile.isPresent()) { + LOG.debug("Whole-file OriginalCode fallback matched remediation {} in {} at lines {}-{}", + instanceId, filename, wholeFile.get().from() + 1, wholeFile.get().to() + 1); + return new LineRange(wholeFile.get().from() + 1, wholeFile.get().to() + 1); + } else { + LOG.debug("Whole-file OriginalCode fallback failed for remediation {} in {}", instanceId, filename); + SkipReason failureReason = priorApplied.isEmpty() + ? SkipReason.SOURCE_CONTEXT_NOT_FOUND + : SkipReason.ANCHOR_DOES_NOT_MATCH; + throw new SkipRemediationException(failureReason, "Anchor not found for file '" + filename + + "'; " + (priorApplied.isEmpty() + ? "file may have changed on disk or context is missing" + : "prior remediation shifted or rewrote the anchor lines this run")); + } + } else { + LOG.debug("Context for remediation {} in {} matched at line {}", instanceId, filename, contextLineFrom + 1); + String originalCodeText = hunk.requiredOriginalCode(); + List originalCodeLine = Arrays.asList(originalCodeText.split("\\r?\\n")); + Optional lineFromTo = fuzzyAnchorLocator.searchOriginalCode(instanceId, filename, originalLines, originalCodeLine, + contextLineFrom, contextLine.size(), contextBefore, contextAfter, projectedFrom, projectedTo); + if (lineFromTo.isEmpty()) { + LOG.debug("Original code search failed for remediation {} in {}; context line={}, original code lines={}, source lines={}", + instanceId, filename, contextLineFrom + 1, originalCodeLine.size(), originalLines.size()); + SkipReason failureReason = priorApplied.isEmpty() + ? SkipReason.ORIGINAL_CODE_NOT_FOUND + : SkipReason.ANCHOR_DOES_NOT_MATCH; + throw new SkipRemediationException(failureReason, "Original code not found for file '" + filename + + "'; " + (priorApplied.isEmpty() + ? "file may have changed on disk" + : "prior remediation altered lines inside this hunk's context window")); + } + int resultLineFrom = lineFromTo.get().from() + 1; + int resultLineTo = lineFromTo.get().to() + 1; + LOG.debug("Original code for remediation {} in {} matched at lines {}-{}", instanceId, filename, resultLineFrom, resultLineTo); + return new LineRange(resultLineFrom, resultLineTo); + } + } + + private void dropDuplicatedBoundaryTokens(List newCodeLines, List originalLines, + int lineFrom, int lineTo, String instanceId, String filename) { + if (newCodeLines.isEmpty()) return; + if (lineFrom > 1 && newCodeLines.size() > 1) { + String lineBefore = originalLines.get(lineFrom - 2); + if (boundaryLinesMatch(lineBefore, newCodeLines.get(0))) { + LOG.debug("Remediation {} for '{}': dropping duplicated leading boundary token in NewCode (matches line {})", + instanceId, filename, lineFrom - 1); + newCodeLines.remove(0); + } + } + if (lineTo < originalLines.size() && newCodeLines.size() > 1) { + String lineAfter = originalLines.get(lineTo); + if (boundaryLinesMatch(lineAfter, newCodeLines.get(newCodeLines.size() - 1))) { + LOG.debug("Remediation {} for '{}': dropping duplicated trailing boundary token in NewCode (matches line {})", + instanceId, filename, lineTo + 1); + newCodeLines.remove(newCodeLines.size() - 1); + } + } + } + + private boolean boundaryLinesMatch(String a, String b) { + if (a == null || b == null) return false; + String normA = a.trim().replaceAll("\\s+", " "); + String normB = b.trim().replaceAll("\\s+", " "); + return !normA.isEmpty() && normA.equals(normB); + } + + private void validateLineRange(int lineFrom, int lineTo, int sourceLineCount, String filename) { + if (lineFrom < 1 || lineTo < lineFrom || lineTo > sourceLineCount) { + throw new SkipRemediationException(SkipReason.REMEDIATION_LINE_RANGE_INVALID, + "Invalid remediation line range " + lineFrom + "-" + lineTo + " for file '" + filename + "'"); + } + } + + /** + * Anchor verification: line-by-line whitespace-insensitive, case-insensitive comparison + * between a slice of the current file and the expected OriginalCode. Matches the same + * normalization ({@code trim().replaceAll("\\s+", " ")}, {@code equalsIgnoreCase}) that + * {@link com.fortify.cli.aviator.util.FuzzyContextSearcher} uses so behaviour is consistent + * between the fast projection path and the fuzzy fallback. + */ + private boolean linesEqualNormalized(List source, int startInclusive, int endInclusive, List expected) { + int len = endInclusive - startInclusive + 1; + if (len != expected.size()) return false; + for (int i = 0; i < len; i++) { + String a = source.get(startInclusive + i).trim().replaceAll("\\s+", " "); + String b = expected.get(i).trim().replaceAll("\\s+", " "); + if (!a.equalsIgnoreCase(b)) return false; + } + return true; + } + + private String detectLineSeparator(String content) { + int crlfIndex = content.indexOf("\r\n"); + int lfIndex = content.indexOf('\n'); + int crIndex = content.indexOf('\r'); + + if (crlfIndex >= 0 && (lfIndex == crlfIndex + 1 || lfIndex < 0) && (crIndex == crlfIndex || crIndex < 0)) { + return "\r\n"; + } + if (lfIndex >= 0 && (crIndex < 0 || lfIndex < crIndex)) { + return "\n"; + } + if (crIndex >= 0) { + return "\r"; + } + return System.lineSeparator(); + } + + private String normalizeLineEndings(String content) { + return content.replace("\r\n", "\n").replace('\r', '\n'); + } + + private String describeLineSeparator(String lineSeparator) { + return switch (lineSeparator) { + case "\r\n" -> "CRLF"; + case "\n" -> "LF"; + case "\r" -> "CR"; + default -> "system"; + }; + } + + /** Encoding-agnostic hash — caller supplies the already-encoded bytes. */ + private String calculateHashBase64Bytes(byte[] bytes, String algorithm) { + if (bytes == null) return ""; + try { + MessageDigest md = MessageDigest.getInstance(algorithm); + return Base64.getEncoder().encodeToString(md.digest(bytes)); + } catch (NoSuchAlgorithmException e) { + throw new AviatorTechnicalException("Hashing algorithm not available: " + algorithm, e); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/classifier/AppliedChangeLedger.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/classifier/AppliedChangeLedger.java new file mode 100644 index 00000000000..64475688129 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/classifier/AppliedChangeLedger.java @@ -0,0 +1,106 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.classifier; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.fortify.cli.aviator.fpr.remediation.model.AppliedChange; + +/** + * Per-FPR offset ledger, NOT shared across artifacts in multi-FPR runs (e.g., --all-open-issues). + * Each RemediationProcessor gets a fresh instance isolated to that artifact's scan. + * + *

Line numbers in AppliedChange are expressed in pristine-file coordinates relative to that + * FPR's audit. Different artifacts are scans of potentially different source revisions, so + * sharing a ledger across artifacts would incorrectly project deltas from one coordinate system + * onto another. Anchor verification (hash checking) is the safety mechanism for multi-FPR runs: + * once an earlier FPR has touched a file, the declared hash no longer matches, so every later + * hunk is written only where its OriginalCode still literally matches. Phase 3 (server-side + * re-audit on code change) is the fundamental fix for multi-FPR consistency. + * + *

Populated when a hunk is written to disk; consulted before applying subsequent hunks to + * detect SUPERSEDED/CONFLICTS and to project declared line ranges through prior edits within + * the same FPR. + */ +public final class AppliedChangeLedger { + private final Map> appliedByFile = new LinkedHashMap<>(); + private final List pendingAppliedChanges = new ArrayList<>(); + + /** + * Committed changes for a file, plus any changes staged so far by the remediation currently + * being processed (not yet committed). Hunks of the same remediation are applied one at a + * time in a single pass ({@code FileWriteCoordinator.processFileChanges}), so a later hunk's + * offset projection must be able to see the shift an earlier hunk in that same remediation + * already staged, not just changes committed by prior remediations. + */ + public List changesFor(Path filePath) { + List committed = appliedByFile.getOrDefault(filePath, List.of()); + List staged = pendingAppliedChanges.stream() + .filter(pac -> pac.filePath().equals(filePath)) + .map(AppliedChangeLedger::fromPending) + .toList(); + if (staged.isEmpty()) { + return committed; + } + List combined = new ArrayList<>(committed); + combined.addAll(staged); + return combined; + } + + /** Stage a hunk this remediation intends to apply. Merged into the ledger on {@link #commitStaged()}. */ + public void stage(PendingAppliedChange pendingAppliedChange) { + pendingAppliedChanges.add(pendingAppliedChange); + } + + /** Discards all currently staged entries (skip/rollback of the whole remediation). */ + public void discardStaged() { + pendingAppliedChanges.clear(); + } + + /** Moves all staged entries into the committed ledger (called only after a successful write) and clears staging. */ + public void commitStaged() { + for (PendingAppliedChange pac : pendingAppliedChanges) { + appliedByFile.computeIfAbsent(pac.filePath(), k -> new ArrayList<>()) + .add(fromPending(pac)); + } + pendingAppliedChanges.clear(); + } + + private static AppliedChange fromPending(PendingAppliedChange pac) { + return AppliedChange.builder() + .originalLineFrom(pac.lineFrom()) + .originalLineTo(pac.lineTo()) + .deltaLines(pac.deltaLines()) + .comparisonCode(pac.comparisonCode()) + .lineNormalizedCode(pac.lineNormalizedCode()) + .build(); + } + + /** + * Accumulated line-delta shift for {@code filePath} from every committed change whose + * original range sits strictly before {@code lineFrom}. + */ + public int projectOffset(Path filePath, int lineFrom) { + int shift = 0; + for (AppliedChange ac : changesFor(filePath)) { + if (ac.originalLineTo() < lineFrom) { + shift += ac.deltaLines(); + } + } + return shift; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/classifier/HunkClassifier.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/classifier/HunkClassifier.java new file mode 100644 index 00000000000..d9eb1754729 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/classifier/HunkClassifier.java @@ -0,0 +1,95 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.classifier; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import com.fortify.cli.aviator.fpr.remediation.exception.SkipRemediationException; +import com.fortify.cli.aviator.fpr.remediation.model.AppliedChange; +import com.fortify.cli.aviator.fpr.remediation.model.FileChange; +import com.fortify.cli.aviator.fpr.remediation.model.Hunk; +import com.fortify.cli.aviator.fpr.remediation.model.HunkOutcome; +import com.fortify.cli.aviator.fpr.remediation.model.Remediation; + +/** + * Pre-classifies each hunk of a {@link Remediation} against the per-run {@link AppliedChangeLedger}. + * Returns {@link HunkOutcome#SUPERSEDED} if a prior applied hunk fully contains the range AND + * its fix content actually covers this hunk's proposed change (normalized, comment/whitespace- + * insensitive substring match); {@link HunkOutcome#POSSIBLY_REMEDIATED} for a fully-nested + * range whose content does NOT match what was actually written (a different fix hidden behind + * a broader one, but the location is still covered); {@link HunkOutcome#CONFLICTS} for a + * partial, non-nested overlap (coverage is genuinely ambiguous); {@link HunkOutcome#APPLIED} + * for no overlap (candidate to attempt). Identity-satisfied hunks whose exact range was written + * by a prior remediation naturally classify as SUPERSEDED, which is semantically correct. + */ +public final class HunkClassifier { + + public List classifyRemediationHunks(Remediation remediation, Path sourceBasePath, AppliedChangeLedger ledger) { + List outcomes = new ArrayList<>(); + for (FileChange fileChange : remediation.fileChanges()) { + Path filePath; + try { + filePath = fileChange.resolve(sourceBasePath); + } catch (SkipRemediationException e) { + outcomes.add(HunkOutcome.APPLIED); + continue; + } + List applied = ledger.changesFor(filePath); + for (Hunk hunk : fileChange.hunks()) { + int from; + int to; + try { + from = hunk.lineFrom(); + to = hunk.lineTo(); + } catch (SkipRemediationException e) { + outcomes.add(HunkOutcome.APPLIED); + continue; + } + String candidateComparisonCode = null; + try { + candidateComparisonCode = hunk.comparisonCode(fileChange.requiredFilename()); + } catch (SkipRemediationException e) { + // Content unavailable for comparison; classifyRange falls back to range-only classification. + } + outcomes.add(classifyRange(from, to, applied, candidateComparisonCode)); + } + } + return outcomes; + } + + /** + * SUPERSEDED if nested in an AppliedChange whose written content actually covers this + * hunk's proposed fix (normalized substring match); POSSIBLY_REMEDIATED if nested but the + * content differs (the sibling fully covers this location, just not proven identical); + * CONFLICTS if there is only a partial, non-nested line overlap (neither range contains + * the other, so coverage is genuinely ambiguous); APPLIED otherwise. When either side's + * content is unavailable ({@code null}) or blank to prove anything, a nested range + * falls back to POSSIBLY_REMEDIATED rather than being assumed SUPERSEDED. + */ + private HunkOutcome classifyRange(int lineFrom, int lineTo, List applied, String candidateComparisonCode) { + for (AppliedChange ac : applied) { + if (ac.coversFully(lineFrom, lineTo)) { + if (ac.contentCovers(candidateComparisonCode, lineFrom, lineTo)) { + return HunkOutcome.SUPERSEDED; + } + return HunkOutcome.POSSIBLY_REMEDIATED; + } + if (ac.overlapsPartially(lineFrom, lineTo)) { + return HunkOutcome.CONFLICTS; + } + } + return HunkOutcome.APPLIED; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/classifier/PendingAppliedChange.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/classifier/PendingAppliedChange.java new file mode 100644 index 00000000000..b714e621f85 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/classifier/PendingAppliedChange.java @@ -0,0 +1,19 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.classifier; + +import java.nio.file.Path; + +public record PendingAppliedChange(Path filePath, int lineFrom, int lineTo, int deltaLines, + String comparisonCode, String lineNormalizedCode) { +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/exception/RemediationCommitException.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/exception/RemediationCommitException.java new file mode 100644 index 00000000000..5d1232c61c0 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/exception/RemediationCommitException.java @@ -0,0 +1,33 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.exception; + +import java.util.List; + +import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; +import com.fortify.cli.aviator.fpr.remediation.writer.RollbackFileWrite; + +public class RemediationCommitException extends AviatorTechnicalException { + private static final long serialVersionUID = 1L; + + private final List rollbacks; + + public RemediationCommitException(String message, Throwable cause, List rollbacks) { + super(message, cause); + this.rollbacks = rollbacks; + } + + public List getRollbacks() { + return rollbacks; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/exception/RollbackRemediationException.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/exception/RollbackRemediationException.java new file mode 100644 index 00000000000..1dc98c2dc76 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/exception/RollbackRemediationException.java @@ -0,0 +1,23 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.exception; + +import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; + +public class RollbackRemediationException extends AviatorTechnicalException { + private static final long serialVersionUID = 1L; + + public RollbackRemediationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/exception/SkipRemediationException.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/exception/SkipRemediationException.java new file mode 100644 index 00000000000..ab379b6516d --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/exception/SkipRemediationException.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.exception; + +import com.fortify.cli.aviator._common.exception.AviatorSimpleException; +import com.fortify.cli.aviator.fpr.remediation.SkipReason; + +public class SkipRemediationException extends AviatorSimpleException { + private static final long serialVersionUID = 1L; + + private final SkipReason reason; + + public SkipRemediationException(SkipReason reason, String message) { + super(message); + this.reason = reason; + } + + public SkipRemediationException(SkipReason reason, String message, Throwable cause) { + super(message, cause); + this.reason = reason; + } + + public SkipReason getReason() { + + return reason; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/AppliedChange.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/AppliedChange.java new file mode 100644 index 00000000000..f37376b2fcc --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/AppliedChange.java @@ -0,0 +1,118 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.model; + +import lombok.Builder; + +/** + * Offset-map entry: records a hunk that was actually written to a file this run, + * in terms of the PRISTINE file's line numbers. {@code deltaLines} is + * (newLineCount - originalLineCount); positive means the file grew, negative means it shrunk. + * + *

Was a record; promoted to a class because the line-range/content comparisons in + * {@code classifyRange} are logic that reads these fields and belongs on the object that owns + * them, not as free functions in the caller. + */ +public final class AppliedChange { + private final int originalLineFrom; + private final int originalLineTo; + private final int deltaLines; + private final String comparisonCode; + private final String[] lineNormalizedContent; + + public AppliedChange(int originalLineFrom, int originalLineTo, int deltaLines, String comparisonCode) { + this(originalLineFrom, originalLineTo, deltaLines, comparisonCode, null); + } + + @Builder + public AppliedChange(int originalLineFrom, int originalLineTo, int deltaLines, String comparisonCode, String lineNormalizedCode) { + this.originalLineFrom = originalLineFrom; + this.originalLineTo = originalLineTo; + this.deltaLines = deltaLines; + this.comparisonCode = comparisonCode; + // Store line-by-line normalized content for offset-anchored comparison (newlines preserved, each line normalized) + if (lineNormalizedCode != null && !lineNormalizedCode.isEmpty()) { + this.lineNormalizedContent = lineNormalizedCode.split("\n", -1); + } else { + this.lineNormalizedContent = null; + } + } + + public int originalLineTo() { + return originalLineTo; + } + + public int deltaLines() { + return deltaLines; + } + + /** True if this change's original range fully contains [lineFrom, lineTo]. */ + public boolean coversFully(int lineFrom, int lineTo) { + return originalLineFrom <= lineFrom && lineTo <= originalLineTo; + } + + /** True if [lineFrom, lineTo] overlaps this change's original range without either side fully containing the other. */ + public boolean overlapsPartially(int lineFrom, int lineTo) { + boolean disjoint = lineTo < originalLineFrom || lineFrom > originalLineTo; + return !disjoint; + } + + /** + * Below this length a normalized comparison code (e.g. {@code "return;"}) is too short and + * generic to prove that a substring hit inside a broader fix's content is the same fix, + * rather than an incidental match. + */ + private static final int MIN_PROVEN_COVERAGE_LENGTH = 8; + + /** + * True if this change's comparison code contains the candidate's comparison code at the + * expected offset (offset-anchored matching). Calculates where the candidate hunk is + * located relative to this broader fix (as an offset of lines), then checks if the candidate + * content appears at that offset within this fix's content. Falls back to false (POSSIBLY_REMEDIATED) + * if the substring appears but not at the expected offset (incidental match, not proof of coverage). + * + *

Unavailable content (either side {@code null}), blank candidates, and too-short + * candidates prove nothing, so coverage is NOT assumed — callers fall back to + * {@code POSSIBLY_REMEDIATED} rather than {@code SUPERSEDED}. + */ + public boolean contentCovers(String candidateComparisonCode, int candidateLineFrom, int candidateLineTo) { + if (candidateComparisonCode == null || comparisonCode == null || lineNormalizedContent == null) { + return false; + } + if (candidateComparisonCode.isBlank() || candidateComparisonCode.length() < MIN_PROVEN_COVERAGE_LENGTH) { + return false; + } + + // Offset-anchored comparison: check if candidate appears at the expected offset within this fix's content + // Calculate the offset: where does candidateLineFrom sit relative to originalLineFrom? + int offsetFromAppliedStart = candidateLineFrom - originalLineFrom; + int candidateLineCount = candidateLineTo - candidateLineFrom + 1; + + // Check if candidate fits at expected offset within applied content + if (offsetFromAppliedStart < 0 || offsetFromAppliedStart + candidateLineCount > lineNormalizedContent.length) { + return false; + } + + // Reconstruct what we expect to see at the candidate's offset + StringBuilder expectedAtOffset = new StringBuilder(); + for (int i = offsetFromAppliedStart; i < offsetFromAppliedStart + candidateLineCount; i++) { + if (i > offsetFromAppliedStart) { + expectedAtOffset.append("\n"); + } + expectedAtOffset.append(lineNormalizedContent[i]); + } + + // Exact match at expected offset (not just substring anywhere) + return expectedAtOffset.toString().equals(candidateComparisonCode); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/FileChange.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/FileChange.java new file mode 100644 index 00000000000..2929eb3b1c7 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/FileChange.java @@ -0,0 +1,54 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.model; + +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.util.List; + +import com.fortify.cli.aviator.fpr.remediation.SkipReason; +import com.fortify.cli.aviator.fpr.remediation.exception.SkipRemediationException; + + +public final class FileChange { + private final String filenameRaw; + private final String hashRaw; + private final List hunks; + + public FileChange(String filenameRaw, String hashRaw, List hunks) { + this.filenameRaw = filenameRaw; + this.hashRaw = hashRaw; + this.hunks = hunks; + } + + public String requiredFilename() { + return RequiredFields.requireText(filenameRaw, "Filename"); + } + + public String requiredHash() { + return RequiredFields.requireText(hashRaw, "Hash"); + } + + public List hunks() { + return hunks; + } + + public Path resolve(Path sourceBasePath) { + try { + return sourceBasePath.resolve(requiredFilename()).normalize(); + } catch (InvalidPathException e) { + throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, + "Invalid filename in remediation: " + requiredFilename(), e); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/Hunk.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/Hunk.java new file mode 100644 index 00000000000..baf41ab29ad --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/Hunk.java @@ -0,0 +1,177 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.model; + +import java.util.Arrays; +import java.util.regex.Pattern; + +import com.fortify.cli.aviator.util.FileTypeLanguageMapperUtil; +import com.fortify.cli.aviator.util.FileUtil; +import com.fortify.cli.aviator.util.LanguageCommentMapperUtil; + +import lombok.Builder; + + +public final class Hunk { + private final String lineFromRaw; + private final String lineToRaw; + private final String contextTextRaw; + private final String contextBeforeRaw; + private final String contextAfterRaw; + private final String originalCodeRaw; + private final String newCodeRaw; + + @Builder + public Hunk(String lineFromRaw, String lineToRaw, String contextTextRaw, String contextBeforeRaw, + String contextAfterRaw, String originalCodeRaw, String newCodeRaw) { + this.lineFromRaw = lineFromRaw; + this.lineToRaw = lineToRaw; + this.contextTextRaw = contextTextRaw; + this.contextBeforeRaw = contextBeforeRaw; + this.contextAfterRaw = contextAfterRaw; + this.originalCodeRaw = originalCodeRaw; + this.newCodeRaw = newCodeRaw; + } + + public int lineFrom() { + return RequiredFields.requireInt(lineFromRaw, "LineFrom"); + } + + public int lineTo() { + return RequiredFields.requireInt(lineToRaw, "LineTo"); + } + + public String requiredContextText() { + return RequiredFields.requireText(contextTextRaw, "Context"); + } + + public int contextBefore() { + return RequiredFields.requireContextAttribute(contextBeforeRaw, "before"); + } + + public int contextAfter() { + return RequiredFields.requireContextAttribute(contextAfterRaw, "after"); + } + + public String requiredOriginalCode() { + return RequiredFields.requireText(originalCodeRaw, "OriginalCode"); + } + + public String requiredNewCode() { + return RequiredFields.requireText(newCodeRaw, "NewCode"); + } + + + public String comparisonCode(String filename) { + String normalizedCode = normalizeProposedCode(requiredNewCode(), filename); + return createComparisonCode(normalizedCode, filename); + } + + /** Normalized code with each line normalized but newlines preserved (for offset-anchored comparison). */ + public String lineNormalizedCode(String filename) { + String normalizedCode = normalizeProposedCode(requiredNewCode(), filename); + return createLineNormalizedCode(normalizedCode, filename); + } + + private static String normalizeProposedCode(String content, String fileName) { + if (content == null) return null; + return trimBlankLines(FileUtil.stripSyntheticLineMarkers(content, fileName, System.lineSeparator())); + } + + private static String createComparisonCode(String normalizedCode, String fileName) { + if (normalizedCode == null) return null; + + String language = FileTypeLanguageMapperUtil.getProgrammingLanguage( + FileUtil.getFileExtension(fileName)); + String commentSymbol = LanguageCommentMapperUtil.getProgrammingLanguageComment(language); + + if ("Unknown".equals(commentSymbol)) { + return normalizeLiteralAliases(normalizedCode).replaceAll("\\s+", ""); + } + + String comparisonCode = normalizedCode; + String closingToken = commentSymbol.equals("" + : commentSymbol.equals("<%--") ? "--%>" : null; + + if (closingToken != null) { + comparisonCode = comparisonCode.replaceAll( + "(?s)" + Pattern.quote(commentSymbol) + ".*?" + Pattern.quote(closingToken), ""); + } else if ("//".equals(commentSymbol)) { + comparisonCode = comparisonCode.replaceAll("(?m)" + Pattern.quote(commentSymbol) + ".*$", "") + .replaceAll("(?s)/\\*.*?\\*/", ""); + } else if ("#".equals(commentSymbol)) { + comparisonCode = comparisonCode.replaceAll("(?m)" + Pattern.quote(commentSymbol) + ".*$", ""); + } + + return normalizeLiteralAliases(comparisonCode).replaceAll("\\s+", ""); + } + + private static String createLineNormalizedCode(String normalizedCode, String fileName) { + if (normalizedCode == null) return null; + + String language = FileTypeLanguageMapperUtil.getProgrammingLanguage( + FileUtil.getFileExtension(fileName)); + String commentSymbol = LanguageCommentMapperUtil.getProgrammingLanguageComment(language); + + if ("Unknown".equals(commentSymbol)) { + return normalizeLinesByPreservingNewlines(normalizeLiteralAliases(normalizedCode)); + } + + String lineNormalizedCode = normalizedCode; + String closingToken = commentSymbol.equals("" + : commentSymbol.equals("<%--") ? "--%>" : null; + + if (closingToken != null) { + lineNormalizedCode = lineNormalizedCode.replaceAll( + "(?s)" + Pattern.quote(commentSymbol) + ".*?" + Pattern.quote(closingToken), ""); + } else if ("//".equals(commentSymbol)) { + lineNormalizedCode = lineNormalizedCode.replaceAll("(?m)" + Pattern.quote(commentSymbol) + ".*$", "") + .replaceAll("(?s)/\\*.*?\\*/", ""); + } else if ("#".equals(commentSymbol)) { + lineNormalizedCode = lineNormalizedCode.replaceAll("(?m)" + Pattern.quote(commentSymbol) + ".*$", ""); + } + + return normalizeLinesByPreservingNewlines(normalizeLiteralAliases(lineNormalizedCode)); + } + + private static String normalizeLinesByPreservingNewlines(String code) { + if (code == null) return null; + String[] lines = code.split("\n", -1); + for (int i = 0; i < lines.length; i++) { + lines[i] = lines[i].replaceAll("\\s+", ""); + } + return String.join("\n", lines); + } + + /** + * Treats semantically-equivalent literal forms as identical for near-identical-fix + * comparison only; never applied to code actually written to source files. + */ + private static String normalizeLiteralAliases(String code) { + if (code == null) return null; + String normalized = code.replaceAll("'\\\\0'", "0"); + normalized = normalized.replaceAll("\\bnullptr\\b", "NULL"); + return normalized; + } + + private static String trimBlankLines(String content) { + String[] lines = content.split("\\R", -1); + int start = 0, end = lines.length - 1; + + while (start <= end && lines[start].isBlank()) start++; + while (end >= start && lines[end].isBlank()) end--; + + return start > end ? "" : + String.join(System.lineSeparator(), Arrays.copyOfRange(lines, start, end + 1)); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/HunkOutcome.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/HunkOutcome.java new file mode 100644 index 00000000000..78186224bbc --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/HunkOutcome.java @@ -0,0 +1,18 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.model; + +/** Per-hunk classification for the state machine. */ +public enum HunkOutcome { + APPLIED, SUPERSEDED, CONFLICTS, POSSIBLY_REMEDIATED +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/Remediation.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/Remediation.java new file mode 100644 index 00000000000..5bda4f830f0 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/Remediation.java @@ -0,0 +1,69 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.model; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import com.fortify.cli.aviator.fpr.remediation.exception.SkipRemediationException; + +public final class Remediation { + private final String instanceId; + private final List fileChanges; + + public Remediation(String instanceId, List fileChanges) { + this.instanceId = instanceId; + this.fileChanges = fileChanges; + } + + public String instanceId() { + return instanceId; + } + + public List fileChanges() { + return fileChanges; + } + + /** + * Widest hunk (lineTo - lineFrom) across all FileChanges/Hunks in this remediation. Used to + * order remediations broader-first so nested narrower fixes classify as SUPERSEDED. + */ + public int maxHunkWidth() { + int max = 0; + for (FileChange fileChange : fileChanges) { + for (Hunk hunk : fileChange.hunks()) { + try { + int from = hunk.lineFrom(); + int to = hunk.lineTo(); + max = Math.max(max, to - from); + } catch (SkipRemediationException ignore) { + // best-effort ordering; malformed hunks fall to the back + } + } + } + return max; + } + + public List createRemediationKeys(Path sourceBasePath) { + List keys = new ArrayList<>(); + for (FileChange fileChange : fileChanges) { + for (Hunk hunk : fileChange.hunks()) { + String filename = fileChange.requiredFilename(); + String comparisonCode = hunk.comparisonCode(filename); + keys.add(RemediationKey.of(fileChange, hunk, sourceBasePath, comparisonCode)); + } + } + return keys; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RemediationDocument.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RemediationDocument.java new file mode 100644 index 00000000000..b3b044282c3 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RemediationDocument.java @@ -0,0 +1,27 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.model; + +import java.util.List; + +public final class RemediationDocument { + private final List remediations; + + public RemediationDocument(List remediations) { + this.remediations = remediations; + } + + public List remediations() { + return remediations; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RemediationKey.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RemediationKey.java new file mode 100644 index 00000000000..fddeabcdf6b --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RemediationKey.java @@ -0,0 +1,25 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.model; + +import java.nio.file.Path; + +public record RemediationKey(Path filePath, int lineFrom, int lineTo, String comparisonCode) { + + public static RemediationKey of(FileChange fileChange, Hunk hunk, Path sourceBasePath, String comparisonCode) { + Path filePath = fileChange.resolve(sourceBasePath); + int lineFrom = hunk.lineFrom(); + int lineTo = hunk.lineTo(); + return new RemediationKey(filePath, lineFrom, lineTo, comparisonCode); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RemediationMetric.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RemediationMetric.java new file mode 100644 index 00000000000..f4fd6cda755 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RemediationMetric.java @@ -0,0 +1,56 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.model; + +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +import lombok.Builder; +import lombok.Getter; +import lombok.experimental.Accessors; + +@Getter +@Accessors(fluent = true) +@Builder +public final class RemediationMetric { + private final int totalRemediations; + private final int appliedRemediations; + private final int identicalRemediations; + private final int supersededRemediations; + private final int possiblyRemediatedRemediations; + private final int skippedRemediations; + private final Set modifiedFiles; + private final Map skippedByReason; + + public static final class RemediationMetricBuilder { + public RemediationMetricBuilder() { + this.modifiedFiles = new LinkedHashSet<>(); + this.skippedByReason = new HashMap<>(); + } + + public RemediationMetricBuilder add(RemediationMetric metric) { + this.totalRemediations += metric.totalRemediations; + this.appliedRemediations += metric.appliedRemediations; + this.identicalRemediations += metric.identicalRemediations; + this.supersededRemediations += metric.supersededRemediations; + this.possiblyRemediatedRemediations += metric.possiblyRemediatedRemediations; + this.skippedRemediations += metric.skippedRemediations; + this.modifiedFiles.addAll(metric.modifiedFiles()); + metric.skippedByReason().forEach((reason, count) -> + this.skippedByReason.merge(reason, count, Integer::sum)); + return this; + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RequiredFields.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RequiredFields.java new file mode 100644 index 00000000000..97c8c3fa2db --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/model/RequiredFields.java @@ -0,0 +1,58 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.model; + +import com.fortify.cli.aviator.fpr.remediation.*; +import com.fortify.cli.aviator.fpr.remediation.exception.SkipRemediationException; + + +public final class RequiredFields { + + private RequiredFields() { + } + + public static String requireText(String value, String elementName) { + if (value == null) { + throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, + "Missing required remediation element '" + elementName + "'"); + } + return value; + } + + public static int requireInt(String value, String elementName) { + String text = requireText(value, elementName); + try { + return Integer.parseInt(text); + } catch (NumberFormatException e) { + throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, + "Invalid integer value for remediation element '" + elementName + "': " + text, e); + } + } + + public static int requireContextAttribute(String value, String attributeName) { + if (value == null || value.isBlank()) { + throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, + "Missing required remediation context attribute '" + attributeName + "'"); + } + try { + int parsedValue = Integer.parseInt(value); + if (parsedValue < 0) { + throw new NumberFormatException("negative value"); + } + return parsedValue; + } catch (NumberFormatException e) { + throw new SkipRemediationException(SkipReason.REMEDIATION_DATA_INVALID, + "Invalid remediation context attribute '" + attributeName + "': " + value, e); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/FileWriteCoordinator.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/FileWriteCoordinator.java new file mode 100644 index 00000000000..58dbaccd1eb --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/FileWriteCoordinator.java @@ -0,0 +1,219 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.writer; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.remediation.SkipReason; +import com.fortify.cli.aviator.fpr.remediation.applier.AppliedChangeResult; +import com.fortify.cli.aviator.fpr.remediation.applier.RemediationApplier; +import com.fortify.cli.aviator.fpr.remediation.classifier.AppliedChangeLedger; +import com.fortify.cli.aviator.fpr.remediation.classifier.PendingAppliedChange; +import com.fortify.cli.aviator.fpr.remediation.exception.RemediationCommitException; +import com.fortify.cli.aviator.fpr.remediation.exception.SkipRemediationException; +import com.fortify.cli.aviator.fpr.remediation.model.FileChange; +import com.fortify.cli.aviator.fpr.remediation.model.Hunk; +import com.fortify.cli.aviator.fpr.remediation.model.Remediation; +import com.fortify.cli.aviator.fpr.remediation.model.RemediationKey; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.DecodeResult; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.SourceDecodeException; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder.SourceEncodeException; + +/** Houses the original prepareFileChanges/processFileChanges/commit/rollback/read/encode logic, unmodified. */ +public final class FileWriteCoordinator { + private static final Logger LOG = LoggerFactory.getLogger(FileWriteCoordinator.class); + + private final ISourceDecoder sourceDecoder; + private final RemediationApplier remediationApplier; + + public FileWriteCoordinator(ISourceDecoder sourceDecoder, RemediationApplier remediationApplier) { + this.sourceDecoder = sourceDecoder; + this.remediationApplier = remediationApplier; + } + + public PreparedFileChanges prepareFileChanges(Remediation remediation, Path sourceBasePath, FVDLMetadata fvdlMetadata, + Set keysToApply, AppliedChangeLedger ledger) { + List fileChanges = remediation.fileChanges(); + if (fileChanges.isEmpty()) { + throw new SkipRemediationException(SkipReason.NO_CHANGES, "No file changes found"); + } + + Map pendingWrites = new LinkedHashMap<>(); + Set appliedKeys = new LinkedHashSet<>(); + // All-or-nothing: if any file's hunk(s) in this remediation can't be applied, the whole + // remediation is skipped rather than partially written to some files but not others. + for (FileChange fileChange : fileChanges) { + processFileChanges(remediation, fileChange, sourceBasePath, fvdlMetadata, pendingWrites, keysToApply, + appliedKeys, ledger); + } + return new PreparedFileChanges(pendingWrites, appliedKeys); + } + + private void processFileChanges(Remediation remediation, FileChange fileChange, Path sourceBasePath, FVDLMetadata fvdlMetadata, + Map pendingWrites, Set keysToApply, Set appliedKeysOut, + AppliedChangeLedger ledger) { + + String instanceId = remediation.instanceId(); + String filename = fileChange.requiredFilename(); + Path filePath = fileChange.resolve(sourceBasePath); + LOG.debug("Processing remediation {} file change for '{}' resolved to '{}'", instanceId, filename, filePath); + + if (!filePath.startsWith(sourceBasePath)) { + throw new SkipRemediationException(SkipReason.SOURCE_FILE_OUTSIDE_SOURCE_DIR, + "Source file resolves outside source directory: " + filename); + } + + if (!isFilePresent(filePath)) { + throw new SkipRemediationException(SkipReason.SOURCE_FILE_MISSING, "Source code file not present at: " + filePath); + } + + String fileHash = fileChange.requiredHash(); + List hunks = fileChange.hunks(); + if (hunks.isEmpty()) { + throw new SkipRemediationException(SkipReason.NO_CHANGES, "No changes found for file: " + filename); + } + SourceFileContent sourceFileContent = getPendingOrSourceContent(filePath, filename, fvdlMetadata, pendingWrites); + Charset sourceEncoding = sourceFileContent.charset(); + LOG.debug("Remediation {} has {} change(s) for '{}' using source encoding {}", instanceId, hunks.size(), filename, + sourceFileContent.encodingSource()); + + String updatedContent = sourceFileContent.content(); + int appliedInThisFile = 0; + int skippedAlreadySatisfied = 0; + for (int k = 0; k < hunks.size(); k++) { + Hunk hunk = hunks.get(k); + String comparisonCode = hunk.comparisonCode(filename); + RemediationKey key = RemediationKey.of(fileChange, hunk, sourceBasePath, comparisonCode); + if (keysToApply != null && !keysToApply.contains(key)) { + LOG.info("Skipping hunk {} of remediation {} in '{}': already applied by prior identical hunk", + k + 1, instanceId, filename); + skippedAlreadySatisfied++; + continue; + } + int declaredLineFrom = hunk.lineFrom(); + int declaredLineTo = hunk.lineTo(); + int linesBeforeChange = updatedContent.split("\n", -1).length; + AppliedChangeResult applyResult = remediationApplier.applyChange(instanceId, filename, filePath, fileHash, sourceEncoding, updatedContent, + hunk, k + 1, ledger); + updatedContent = applyResult.updatedContent(); + // Stage this hunk into the per-run offset map using the *actual* coordinates where it was written, + // not the declared ones (which may differ if the applier relocated it via offset projection or + // fuzzy anchor). The ledger must reflect reality for later hunks to correctly project offsets + // and for the classifier to compare against the right region. + int actualLineFrom = applyResult.actualLineFrom(); + int actualLineTo = applyResult.actualLineTo(); + int linesAfterChange = updatedContent.split("\n", -1).length; + int delta = linesAfterChange - linesBeforeChange; + String lineNormalizedCode = hunk.lineNormalizedCode(filename); + ledger.stage(new PendingAppliedChange(filePath, actualLineFrom, actualLineTo, delta, comparisonCode, lineNormalizedCode)); + appliedKeysOut.add(key); + appliedInThisFile++; + } + if (appliedInThisFile == 0) { + LOG.debug("Remediation {} produced no new hunks for '{}' ({} already satisfied); no write staged", + instanceId, filename, skippedAlreadySatisfied); + return; + } + byte[] updatedBytes = encodeSourceFile(updatedContent, sourceEncoding, filename); + + pendingWrites.put(filePath, new PendingFileWrite(filename, filePath, updatedContent, sourceEncoding, + sourceFileContent.encodingSource(), updatedBytes)); + LOG.debug("Staged remediation {} for '{}' using source encoding {}; changes={}, encodedBytes={}", instanceId, filename, + sourceFileContent.encodingSource(), hunks.size(), updatedBytes.length); + } + + public void commitRemediationWrites(String instanceId, Map pendingWrites, Set modifiedFiles) + throws RemediationCommitException { + List rollbacks = new ArrayList<>(); + for (PendingFileWrite pendingWrite : pendingWrites.values()) { + try { + byte[] originalBytes = Files.readAllBytes(pendingWrite.filePath()); + rollbacks.add(new RollbackFileWrite(pendingWrite.filename(), pendingWrite.filePath(), originalBytes)); + LOG.debug("Writing remediation {} to '{}' using staged bytes; encodedBytes={}", instanceId, pendingWrite.filename(), + pendingWrite.updatedBytes().length); + Files.write(pendingWrite.filePath(), pendingWrite.updatedBytes()); + } catch (Exception e) { + throw new RemediationCommitException("Error writing source code file '" + pendingWrite.filename() + "'", e, rollbacks); + } + } + + for (PendingFileWrite pendingWrite : pendingWrites.values()) { + modifiedFiles.add(pendingWrite.filename()); + LOG.info("Remediation applied for {} in file {}", instanceId, pendingWrite.filename()); + } + } + + public void rollbackRemediationWrites(String instanceId, List rollbacks) { + for (RollbackFileWrite rollback : rollbacks) { + try { + Files.write(rollback.filePath(), rollback.originalBytes()); + LOG.warn("Rolled back remediation {} changes for '{}' after write failure", instanceId, rollback.filename()); + } catch (IOException rollbackException) { + LOG.error("Failed to roll back remediation {} changes for '{}'", instanceId, rollback.filename(), rollbackException); + throw new com.fortify.cli.aviator.fpr.remediation.exception.RollbackRemediationException( + "Failed to roll back remediation changes for '" + rollback.filename() + + "'. Source files may be partially modified; inspect the source tree before retrying", rollbackException); + } + } + } + + private SourceFileContent getPendingOrSourceContent(Path filePath, String filename, FVDLMetadata fvdlMetadata, + Map pendingWrites) { + PendingFileWrite pendingWrite = pendingWrites.get(filePath); + return pendingWrite == null + ? readSourceFile(filePath, filename, fvdlMetadata) + : new SourceFileContent(pendingWrite.content(), pendingWrite.charset(), pendingWrite.encodingSource()); + } + + private boolean isFilePresent(Path path) { + return Files.exists(path) && Files.isRegularFile(path); + } + + private SourceFileContent readSourceFile(Path filePath, String filename, FVDLMetadata fvdlMetadata) { + try { + byte[] sourceBytes = Files.readAllBytes(filePath); + // Metadata may be null (FVDL missing); FPR candidate fails and other encodings are tried. + DecodeResult decodeResult = sourceDecoder.decode(sourceBytes, filename, fvdlMetadata); + LOG.debug("Strict decoded '{}' using {}; sourceBytes={}, decodedChars={}", filename, decodeResult.source(), sourceBytes.length, + decodeResult.content().length()); + return new SourceFileContent(decodeResult.content(), decodeResult.charset(), decodeResult.source()); + } catch (SourceDecodeException e) { + throw new SkipRemediationException(SkipReason.SOURCE_DECODE_FAILED, e.getMessage(), e); + } catch (IOException e) { + throw new SkipRemediationException(SkipReason.SOURCE_READ_FAILED, "Error reading source code file '" + filePath + "'", e); + } + } + + private byte[] encodeSourceFile(String content, Charset charset, String filename) { + try { + return SourceEncoder.encode(content, charset, filename); + } catch (SourceEncodeException e) { + throw new SkipRemediationException(SkipReason.REMEDIATION_ENCODE_FAILED, e.getMessage(), e); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/PendingFileWrite.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/PendingFileWrite.java new file mode 100644 index 00000000000..de21b337298 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/PendingFileWrite.java @@ -0,0 +1,20 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.writer; + +import java.nio.charset.Charset; +import java.nio.file.Path; + +public record PendingFileWrite(String filename, Path filePath, String content, Charset charset, String encodingSource, + byte[] updatedBytes) { +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/PreparedFileChanges.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/PreparedFileChanges.java new file mode 100644 index 00000000000..da58b649519 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/PreparedFileChanges.java @@ -0,0 +1,23 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.writer; + +import java.nio.file.Path; +import java.util.Map; +import java.util.Set; + +import com.fortify.cli.aviator.fpr.remediation.model.RemediationKey; + + +public record PreparedFileChanges(Map pendingWrites, Set appliedKeys) { +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/RollbackFileWrite.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/RollbackFileWrite.java new file mode 100644 index 00000000000..c304ee8aa60 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/RollbackFileWrite.java @@ -0,0 +1,18 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.writer; + +import java.nio.file.Path; + +public record RollbackFileWrite(String filename, Path filePath, byte[] originalBytes) { +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/SourceFileContent.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/SourceFileContent.java new file mode 100644 index 00000000000..abc50e7d604 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/writer/SourceFileContent.java @@ -0,0 +1,40 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.writer; + +import java.nio.charset.Charset; + + +public final class SourceFileContent { + private final String content; + private final Charset charset; + private final String encodingSource; + + public SourceFileContent(String content, Charset charset, String encodingSource) { + this.content = content; + this.charset = charset; + this.encodingSource = encodingSource; + } + + public String content() { + return content; + } + + public Charset charset() { + return charset; + } + + public String encodingSource() { + return encodingSource; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/xmlprocessor/RemediationDocumentMapper.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/xmlprocessor/RemediationDocumentMapper.java new file mode 100644 index 00000000000..7e7d1a75167 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/xmlprocessor/RemediationDocumentMapper.java @@ -0,0 +1,96 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.xmlprocessor; + +import java.util.ArrayList; +import java.util.List; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +import com.fortify.cli.aviator.fpr.remediation.model.FileChange; +import com.fortify.cli.aviator.fpr.remediation.model.Hunk; +import com.fortify.cli.aviator.fpr.remediation.model.Remediation; +import com.fortify.cli.aviator.fpr.remediation.model.RemediationDocument; + +/** + * Phase 2: maps a DOM {@link Document} into the domain model, without any additional logic — + * no normalization, no comparison codes, no classification, and (deliberately) no validation: + * fields whose element/attribute is absent are stored as {@code null}/blank rather than + * throwing, so that phase 3 code can validate lazily at the exact point of use, exactly as the + * original single-class implementation did. + */ +public final class RemediationDocumentMapper { + private static final String NAMESPACE_URI = "xmlns://www.fortify.com/schema/remediations"; + + public RemediationDocument map(Document remediationDoc) { + NodeList remediationNodes = remediationDoc.getElementsByTagNameNS(NAMESPACE_URI, "Remediation"); + List remediations = new ArrayList<>(); + for (int i = 0; i < remediationNodes.getLength(); i++) { + remediations.add(mapRemediation((Element) remediationNodes.item(i))); + } + return new RemediationDocument(remediations); + } + + private Remediation mapRemediation(Element remediationElement) { + String instanceId = remediationElement.getAttribute("instanceId"); + NodeList fileChangesNodes = remediationElement.getElementsByTagNameNS(NAMESPACE_URI, "FileChanges"); + List fileChanges = new ArrayList<>(); + for (int i = 0; i < fileChangesNodes.getLength(); i++) { + fileChanges.add(mapFileChange((Element) fileChangesNodes.item(i))); + } + return new Remediation(instanceId, fileChanges); + } + + private FileChange mapFileChange(Element fileChangesElement) { + String filename = optionalElementText(fileChangesElement, "Filename"); + String hash = optionalElementText(fileChangesElement, "Hash"); + NodeList changeNodes = fileChangesElement.getElementsByTagNameNS(NAMESPACE_URI, "Change"); + List hunks = new ArrayList<>(); + for (int i = 0; i < changeNodes.getLength(); i++) { + hunks.add(mapHunk((Element) changeNodes.item(i))); + } + return new FileChange(filename, hash, hunks); + } + + private Hunk mapHunk(Element changeElement) { + String lineFrom = optionalElementText(changeElement, "LineFrom"); + String lineTo = optionalElementText(changeElement, "LineTo"); + Element contextElement = optionalElement(changeElement, "Context"); + String contextText = contextElement == null ? null : contextElement.getTextContent(); + String contextBefore = contextElement == null ? null : contextElement.getAttribute("before"); + String contextAfter = contextElement == null ? null : contextElement.getAttribute("after"); + String originalCode = optionalElementText(changeElement, "OriginalCode"); + String newCode = optionalElementText(changeElement, "NewCode"); + return Hunk.builder() + .lineFromRaw(lineFrom) + .lineToRaw(lineTo) + .contextTextRaw(contextText) + .contextBeforeRaw(contextBefore) + .contextAfterRaw(contextAfter) + .originalCodeRaw(originalCode) + .newCodeRaw(newCode) + .build(); + } + + private String optionalElementText(Element parent, String elementName) { + Element element = optionalElement(parent, elementName); + return element == null ? null : element.getTextContent(); + } + + private Element optionalElement(Element parent, String elementName) { + NodeList nodes = parent.getElementsByTagNameNS(NAMESPACE_URI, elementName); + return nodes.getLength() == 0 ? null : (Element) nodes.item(0); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/xmlprocessor/RemediationXmlReader.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/xmlprocessor/RemediationXmlReader.java new file mode 100644 index 00000000000..b81cf5ba87c --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/remediation/xmlprocessor/RemediationXmlReader.java @@ -0,0 +1,51 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.xmlprocessor; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; + +/** Phase 1: parses {@code remediations.xml} into a DOM {@link Document}. No mapping, no business logic. */ +public final class RemediationXmlReader { + private static final Logger LOG = LoggerFactory.getLogger(RemediationXmlReader.class); + + public Document read(Path remediationPath) { + try (InputStream remediationStream = Files.newInputStream(remediationPath)) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); + factory.setXIncludeAware(false); + factory.setExpandEntityReferences(false); + DocumentBuilder builder = factory.newDocumentBuilder(); + return builder.parse(remediationStream); + } catch (ParserConfigurationException | SAXException | IOException e) { + LOG.error("Error parsing remediations.xml file: {}", remediationPath, e); + throw new AviatorTechnicalException("Error processing remediation.xml file.", e); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FileUtil.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FileUtil.java index 5a50574d370..09a3526126e 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FileUtil.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FileUtil.java @@ -21,6 +21,7 @@ import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.Comparator; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; import java.util.zip.ZipInputStream; @@ -30,6 +31,7 @@ import com.fortify.cli.common.exception.FcliTechnicalException; + public final class FileUtil { private static final Logger LOG = LoggerFactory.getLogger(FileTypeLanguageMapperUtil.class); @@ -130,4 +132,55 @@ public static void writeStringToFile(Path filePath, String content, boolean over throw new FcliTechnicalException("Error writing to file " + absolutePath, e); } } -} \ No newline at end of file + + /** + * Canonical form for file hashing. Normalises line endings to LF and strips a single + * trailing newline. Both the audit side (writing the hash into remediations.xml) and the + * apply side (verifying it) must call this before hashing so the two sides agree + * byte-for-byte regardless of the OS that ran the audit or whether the file had a + * trailing newline on disk. Callers hash the UTF-8 bytes of the returned string. + */ + public static String canonicalizeForHash(String content) { + if (content == null) return ""; + String normalized = content.replace("\r\n", "\n").replace('\r', '\n'); + if (normalized.endsWith("\n")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + return normalized; + } + + public static String stripSyntheticLineMarkers(String content, String fileName) { + return stripSyntheticLineMarkers(content, fileName, "\n"); + } + + /** Same as {@link #stripSyntheticLineMarkers(String, String)}, but joins with a caller-chosen line separator. */ + public static String stripSyntheticLineMarkers(String content, String fileName, String lineSeparator) { + if (content == null || content.isEmpty()) { + return content; + } + String language = FileTypeLanguageMapperUtil.getProgrammingLanguage(getFileExtension(fileName)); + String commentSymbol = LanguageCommentMapperUtil.getProgrammingLanguageComment(language); + String stripped = content; + if (!"Unknown".equals(commentSymbol)) { + String closingToken = commentSymbol.equals("" + : commentSymbol.equals("<%--") ? "--%>" + : null; + Pattern markerPattern = Pattern.compile( + "[ \\t]*" + Pattern.quote(commentSymbol) + " L\\d+" + + (closingToken != null ? "[ \\t]*" + Pattern.quote(closingToken) : "") + + "[ \\t]*$"); + String[] lines = content.split("\\R", -1); + StringBuilder result = new StringBuilder(); + for (int i = 0; i < lines.length; i++) { + Matcher matcher = markerPattern.matcher(lines[i]); + result.append(matcher.find() ? lines[i].substring(0, matcher.start()) : lines[i]); + if (i < lines.length - 1) { + result.append(lineSeparator); + } + } + stripped = result.toString(); + } + return stripped; + } + +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FprHandle.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FprHandle.java index 0cca7e377dc..4b10fcab4b7 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FprHandle.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FprHandle.java @@ -153,9 +153,13 @@ public void close() throws IOException { */ public synchronized Map getSourceFileMap() { if (sourceFileMap == null) { - sourceFileMap = Files.exists(zipfs.getPath("/webinspect.xml")) + //03358915/OCTCR11A2429097 fix for audit issue + boolean dastOnly = Files.exists(zipfs.getPath("/webinspect.xml")) + && !Files.exists(zipfs.getPath("/audit.fvdl")); + sourceFileMap = dastOnly ? new ConcurrentHashMap<>() : loadSourceFileMap(); + //03358915/OCTCR11A2429097 fix for audit issue } return sourceFileMap; } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FuzzyContextSearcher.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FuzzyContextSearcher.java index 4a4b8e6dd51..7c698e8d860 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FuzzyContextSearcher.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/FuzzyContextSearcher.java @@ -16,6 +16,8 @@ import java.util.ArrayList; import java.util.List; +import com.fortify.cli.aviator.fpr.remediation.applier.LineRange; + public class FuzzyContextSearcher { /** @@ -27,11 +29,6 @@ public class FuzzyContextSearcher { * @return The line number (0-based) in sourceFile where context starts, or -1 if not found. */ - public static int fuzzySearchContext(List sourceLines, List contextLines, int maxMismatches) throws IOException { - List matches = fuzzySearchContextMatches(sourceLines, contextLines, maxMismatches); - return matches.isEmpty() ? -1 : matches.get(0); - } - public static List fuzzySearchContextMatches(List sourceLines, List contextLines, int maxMismatches) throws IOException { List normalizedSource = normalizeLines(sourceLines); @@ -92,9 +89,11 @@ private static Integer findContextMatchStart(List normalizedSource, List return contextIndex == normalizedContext.size() && mismatchCount <= maxMismatches ? startIndex : null; } - public static int[] fuzzySearchOriginalCode(List sourceLines, List originalCodeLine, int maxMismatches, int startIndex) { + /** Returns every {@code LineRange} match found, so callers can detect ambiguous (multiple-candidate) matches. */ + public static List fuzzySearchOriginalCodeMatches(List sourceLines, List originalCodeLine, int maxMismatches, int startIndex) { List normalizedSource = normalizeLines(sourceLines); List normalizedOriginalCode = normalizeLines(originalCodeLine); + List matches = new ArrayList<>(); for (int i = Math.max(0, startIndex); i < normalizedSource.size(); i++) { if (normalizedSource.get(i).isEmpty()) { @@ -103,11 +102,11 @@ public static int[] fuzzySearchOriginalCode(List sourceLines, List normalizedSource, List normalizedOriginalCode, int maxMismatches, diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessorTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessorTest.java index fe171bb4b71..da9a03ca5b9 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessorTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessorTest.java @@ -17,13 +17,23 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.io.TempDir; +import com.fortify.cli.aviator.fpr.remediation.RemediationProcessor; +import com.fortify.cli.aviator.fpr.remediation.model.*; +import com.fortify.cli.aviator.util.FileUtil; import com.fortify.cli.aviator.util.FprHandle; class RemediationProcessorTest { @@ -32,13 +42,40 @@ class RemediationProcessorTest { @TempDir Path tempDir; + /** + * Fix #2 (exact-line disambiguation): two identical context blocks are ambiguous on their + * own, but the declared/projected LineFrom lands exactly on the first occurrence, so it + * resolves deterministically instead of being skipped as ambiguous. + */ @Test - void skipsAmbiguousContextWithoutChangingSource() throws Exception { + void resolvesAmbiguousContextByExactDeclaredPosition() throws Exception { String originalSource = "before\nTARGET\nafter\nbefore\nTARGET\nafter\n"; Path sourceFile = writeSourceFile(originalSource); Path fprPath = createRemediationFpr(2, 2, 1, 1, "before\ntarget\nafter", "TARGET", "REPLACED"); - RemediationProcessor.RemediationMetric metric; + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(1, metric.totalRemediations()); + assertEquals(1, metric.appliedRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals(Map.of(), metric.skippedByReason()); + assertEquals("before\nREPLACED\nafter\nbefore\nTARGET\nafter\n", Files.readString(sourceFile)); + } + + /** + * Fix #2 must not guess: when the declared/projected position doesn't exactly match any of + * the ambiguous candidates, it still throws SOURCE_CONTEXT_AMBIGUOUS rather than picking one. + */ + @Test + void skipsAmbiguousContextWhenDeclaredPositionMatchesNoCandidate() throws Exception { + String originalSource = "before\nTARGET\nafter\nbefore\nTARGET\nafter\n"; + Path sourceFile = writeSourceFile(originalSource); + Path fprPath = createRemediationFpr(99, 99, 1, 1, "before\ntarget\nafter", "TARGET", "REPLACED"); + + RemediationMetric metric; try (FprHandle fprHandle = new FprHandle(fprPath)) { metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); } @@ -51,13 +88,12 @@ void skipsAmbiguousContextWithoutChangingSource() throws Exception { metric.skippedByReason()); assertEquals(originalSource, Files.readString(sourceFile)); } - @Test void appliesRemediationWhenContextMatchesOnce() throws Exception { Path sourceFile = writeSourceFile("before\nTARGET\nafter\n"); Path fprPath = createRemediationFpr(2, 2, 1, 1, "before\ntarget\nafter", "TARGET", "REPLACED"); - RemediationProcessor.RemediationMetric metric; + RemediationMetric metric; try (FprHandle fprHandle = new FprHandle(fprPath)) { metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); } @@ -72,7 +108,7 @@ void appliesOriginalCodeAfterLeadingContextLines() throws Exception { Path sourceFile = writeSourceFile("TARGET\nkeep\nTARGET\nafter\n"); Path fprPath = createRemediationFpr(3, 3, 2, 1, "TARGET\nkeep\nTARGET\nafter", "TARGET", "REPLACED"); - RemediationProcessor.RemediationMetric metric; + RemediationMetric metric; try (FprHandle fprHandle = new FprHandle(fprPath)) { metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); } @@ -86,7 +122,7 @@ void appliesRemediationWhenContextStartsWithBlankLine() throws Exception { Path sourceFile = writeSourceFile("header\n\nTARGET\nafter\n"); Path fprPath = createRemediationFpr(3, 3, 1, 1, "\nTARGET\nafter", "TARGET", "REPLACED"); - RemediationProcessor.RemediationMetric metric; + RemediationMetric metric; try (FprHandle fprHandle = new FprHandle(fprPath)) { metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); } @@ -95,8 +131,730 @@ void appliesRemediationWhenContextStartsWithBlankLine() throws Exception { assertEquals("header\n\nREPLACED\nafter\n", Files.readString(sourceFile)); } + @Test + void nestedRemediationWithDifferentContentIsPossiblyRemediated() throws Exception { + Path sourceFile = writeSourceFile("before\nTARGET\nafter\n"); + Path fprPath = createRemediationFpr(List.of( + new RemediationSpec("wide-fix", 1, 3, 0, 0, "before\nTARGET\nafter", + "before\nTARGET\nafter", "wideline1\nwideline2\nwideline3"), + new RemediationSpec("narrow-fix", 2, 2, 1, 1, "before\ntarget\nafter", + "TARGET", "NARROW_DIFFERENT"))); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(2, metric.totalRemediations()); + assertEquals(1, metric.appliedRemediations()); + assertEquals(1, metric.possiblyRemediatedRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals(Map.of(), metric.skippedByReason()); + assertEquals("wideline1\nwideline2\nwideline3\n", Files.readString(sourceFile)); + } + + /** + * Fix #1 (boundary-token duplication): NewCode repeats the line immediately before LineFrom + * verbatim as its first line; that duplicate must be dropped rather than doubling the line. + */ + @Test + void dropsDuplicatedLeadingBoundaryLineInNewCode() throws Exception { + Path sourceFile = writeSourceFile("line1\nline2\nline3\n"); + Path fprPath = createRemediationFpr(2, 2, 1, 1, "line1\nline2\nline3", "line2", "line1\nreplaced2"); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(1, metric.appliedRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals("line1\nreplaced2\nline3\n", Files.readString(sourceFile)); + } + + /** + * Fix #1 (boundary-token duplication): NewCode repeats the line immediately after LineTo + * verbatim as its last line; that duplicate must be dropped rather than doubling the line. + */ + @Test + void dropsDuplicatedTrailingBoundaryLineInNewCode() throws Exception { + Path sourceFile = writeSourceFile("line1\nline2\nline3\n"); + Path fprPath = createRemediationFpr(2, 2, 1, 1, "line1\nline2\nline3", "line2", "replaced2\nline3"); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(1, metric.appliedRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals("line1\nreplaced2\nline3\n", Files.readString(sourceFile)); + } + + /** + * Regression test: the offset ledger must record the delta actually spliced into the file + * (post boundary-dedup), not the raw NewCode line count. A prior hunk in this file drops a + * duplicated boundary line (fix #1), shrinking the file by one more line than NewCode's raw + * length implies. A later hunk in the same file targets the second of two identical "TARGET" + * blocks; only a correctly-shifted projection lands exactly on it. With the pre-fix (buggy) + * delta, the projected position misses, the fuzzy fallback's context match is ambiguous + * between the two blocks, and neither lands on the declared/projected position either - + * causing an incorrect skip instead of resolving to the second occurrence. + */ + @Test + void offsetLedgerAccountsForDedupWhenProjectingLaterHunkInSameFile() throws Exception { + String originalSource = "line0\nhead1\nhead2\nhead3\nbefore\nTARGET\nafter\nbefore\nTARGET\nafter\ntail\n"; + Path sourceFile = writeSourceFile(originalSource); + Path fprPath = createRemediationFpr(List.of( + new RemediationSpec("hunkA", 2, 4, 1, 1, "line0\nhead1\nhead2\nhead3\nbefore", + "head1\nhead2\nhead3", "line0\nreplacedHead"), + new RemediationSpec("hunkB", 9, 9, 1, 1, "before\ntarget\nafter", "TARGET", "REPLACED"))); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(2, metric.totalRemediations()); + assertEquals(2, metric.appliedRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals(Map.of(), metric.skippedByReason()); + assertEquals("line0\nreplacedHead\nbefore\nTARGET\nafter\nbefore\nREPLACED\nafter\ntail\n", + Files.readString(sourceFile)); + } + + /** + * Fix #1 (boundary-token duplication): NewCode repeats BOTH the line before LineFrom and the + * line after LineTo verbatim in the same hunk. Both duplicates must be dropped, not just one - + * dropDuplicatedBoundaryTokens checks leading and trailing independently, so this proves + * neither check clobbers or is skipped because of the other having already mutated the list. + */ + @Test + void dropsBothDuplicatedBoundaryLinesWhenNewCodeRepeatsBothNeighbors() throws Exception { + Path sourceFile = writeSourceFile("line1\nline2\nline3\n"); + Path fprPath = createRemediationFpr(2, 2, 1, 1, "line1\nline2\nline3", "line2", + "line1\nreplaced2\nline3"); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(1, metric.appliedRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals("line1\nreplaced2\nline3\n", Files.readString(sourceFile)); + } + + /** + * Fix #2 (exact-line disambiguation) must also cover OriginalCode search, not just Context + * search: two identical single-line OriginalCode matches inside the context window are + * ambiguous on their own, but the declared/projected LineFrom lands exactly on the second + * occurrence, so it resolves deterministically instead of being skipped as ambiguous. + */ + @Test + void resolvesAmbiguousOriginalCodeByExactDeclaredPosition() throws Exception { + String originalSource = "before\nTARGET\nTARGET\nafter\n"; + Path sourceFile = writeSourceFile(originalSource); + Path fprPath = createRemediationFpr(3, 3, 1, 1, "before\nTARGET\nTARGET\nafter", "TARGET", "REPLACED"); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(1, metric.appliedRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals("before\nTARGET\nREPLACED\nafter\n", Files.readString(sourceFile)); + } + + /** + * Companion to the above: when the declared/projected position matches none of the ambiguous + * OriginalCode candidates, it must throw ORIGINAL_CODE_AMBIGUOUS rather than guessing one. + */ + @Test + void skipsAmbiguousOriginalCodeWhenDeclaredPositionMatchesNoCandidate() throws Exception { + String originalSource = "before\nTARGET\nTARGET\nafter\n"; + Path sourceFile = writeSourceFile(originalSource); + Path fprPath = createRemediationFpr(99, 99, 1, 1, "before\nTARGET\nTARGET\nafter", "TARGET", "REPLACED"); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(0, metric.appliedRemediations()); + assertEquals(1, metric.skippedRemediations()); + assertEquals(Map.of("Original code matched multiple locations", 1), metric.skippedByReason()); + assertEquals(originalSource, Files.readString(sourceFile)); + } + + /** + * Phase 2 "make the hash check work": when the declared Hash matches the canonical form of + * the file content, the change is applied directly at the declared line range with NO context + * or OriginalCode search at all. Context and OriginalCode here are deliberately garbage text + * that appears nowhere in the source - if the hash-match fast path were not actually short- + * circuiting the fuzzy search, this remediation would be skipped as not-found. + */ + @Test + void appliesRemediationViaCanonicalHashMatchWithoutContextOrOriginalCodeSearch() throws Exception { + String originalSource = "before\nTARGET\nafter\n"; + Path sourceFile = writeSourceFile(originalSource); + String canonicalHash = sha256Base64(FileUtil.canonicalizeForHash(originalSource) + .getBytes(StandardCharsets.UTF_8)); + Path fprPath = createRemediationFprWithHash(2, 2, 1, 1, + "garbage-context-not-in-file\nmore-garbage\nstill-garbage", "GARBAGE-CODE-NOT-IN-FILE", + "REPLACED", canonicalHash); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(1, metric.appliedRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals("before\nREPLACED\nafter\n", Files.readString(sourceFile)); + } + + /** + * SUPERSEDED: a broader prior fix's actually-written content already contains the narrower + * candidate's proposed replacement (normalized). The narrower remediation must be recognized + * as superseded and not re-applied/re-searched at all. + */ + @Test + void supersededRemediationWithMatchingContentIsNotReapplied() throws Exception { + Path sourceFile = writeSourceFile("before\nTARGET\nafter\n"); + Path fprPath = createRemediationFpr(List.of( + new RemediationSpec("wide-fix", 1, 3, 0, 0, "before\nTARGET\nafter", + "before\nTARGET\nafter", "before\nREPLACED\nafter"), + new RemediationSpec("narrow-fix", 2, 2, 1, 1, "before\ntarget\nafter", + "TARGET", "REPLACED"))); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(2, metric.totalRemediations()); + assertEquals(1, metric.appliedRemediations()); + assertEquals(1, metric.supersededRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals(Map.of(), metric.skippedByReason()); + assertEquals("before\nREPLACED\nafter\n", Files.readString(sourceFile)); + } + + /** + * CONFLICTS: two remediations with a partial, non-nested line overlap (neither range contains + * the other) is genuinely ambiguous coverage. Fix #2's "no heuristic guessing" philosophy + * extends here too: the later, overlapping remediation must be skipped with + * CONFLICTS_WITH_ANOTHER_FIX rather than guessed at. + */ + @Test + void conflictingOverlappingRemediationIsSkippedNotGuessed() throws Exception { + Path sourceFile = writeSourceFile("line1\nline2\nline3\nline4\nline5\n"); + Path fprPath = createRemediationFpr(List.of( + new RemediationSpec("fix-A", 2, 3, 1, 1, "line1\nline2\nline3\nline4", + "line2\nline3", "A2\nA3"), + new RemediationSpec("fix-B", 3, 4, 1, 1, "line2\nline3\nline4\nline5", + "line3\nline4", "B3\nB4"))); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(2, metric.totalRemediations()); + assertEquals(1, metric.appliedRemediations()); + assertEquals(1, metric.skippedRemediations()); + assertEquals(Map.of("Conflicts with another fix", 1), metric.skippedByReason()); + assertEquals("line1\nA2\nA3\nline4\nline5\n", Files.readString(sourceFile)); + } + + /** + * Mixed outcome within a single remediation: one file-change is a valid APPLY candidate + * while a sibling file-change in the SAME remediation conflicts with an already-applied + * prior fix. Remediations apply atomically, so the whole remediation must be rejected as + * CONFLICTS_WITH_ANOTHER_FIX as soon as any hunk is CONFLICTS, rather than falling through + * to the applier's best-effort offset/fuzzy-anchor fallback for the conflicting hunk while + * silently applying the valid one. + */ + @Test + void mixedOutcomeRemediationWithOneConflictingHunkIsSkippedEntirely() throws Exception { + Path sharedFile = writeSourceFile("Shared.java", "s1\ns2\ns3\ns4\ns5\n"); + Path exampleFile = writeSourceFile("Example.java", "before\nTARGET\nafter\n"); + + LinkedHashMap> remediations = new LinkedHashMap<>(); + remediations.put("fix-prior", List.of( + new FileChangeSpec("Shared.java", 2, 3, 1, 1, "s1\ns2\ns3\ns4", "s2\ns3", "P2\nP3"))); + remediations.put("fix-mixed", List.of( + new FileChangeSpec("Example.java", 2, 2, 1, 1, "before\nTARGET\nafter", "TARGET", "REPLACED"), + new FileChangeSpec("Shared.java", 3, 4, 1, 1, "s2\ns3\ns4\ns5", "s3\ns4", "M3\nM4"))); + Path fprPath = createRemediationFprWithRemediations(remediations); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(2, metric.totalRemediations()); + assertEquals(1, metric.appliedRemediations()); + assertEquals(1, metric.skippedRemediations()); + assertEquals(Map.of("Conflicts with another fix", 1), metric.skippedByReason()); + assertEquals("s1\nP2\nP3\ns4\ns5\n", Files.readString(sharedFile)); + assertEquals("before\nTARGET\nafter\n", Files.readString(exampleFile)); + } + + /** + * Near-identical recognition (Phase 2): two remediations at the same location whose NewCode + * differs only by a trailing Java comment normalize to the same comparisonCode. The second + * must be recognized as fully identical to the first and counted once, not re-applied and not + * leaving any trace of its own NewCode text in the file. + */ + @Test + void fullyIdenticalNearIdenticalRemediationIsCountedOnceNotReapplied() throws Exception { + Path sourceFile = writeSourceFile("before\nTARGET\nafter\n"); + Path fprPath = createRemediationFpr(List.of( + new RemediationSpec("fix-1", 2, 2, 1, 1, "before\ntarget\nafter", "TARGET", "REPLACED"), + new RemediationSpec("fix-2", 2, 2, 1, 1, "before\ntarget\nafter", "TARGET", "REPLACED // note"))); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(2, metric.totalRemediations()); + assertEquals(1, metric.appliedRemediations()); + assertEquals(1, metric.identicalRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals("before\nREPLACED\nafter\n", Files.readString(sourceFile)); + } + + /** + * All-or-nothing multi-file remediations: one remediation touching two files where one + * file's hunk cannot be located must skip the WHOLE remediation, not just that file. Neither + * file may end up written - a half-applied remediation is worse than one left unapplied. + */ + @Test + void multiFileRemediationSkipsEntirelyWhenAnyFileFails() throws Exception { + Path badFile = writeSourceFile("Bad.java", "one\ntwo\nthree\n"); + Path goodFile = writeSourceFile("Good.java", "before\nTARGET\nafter\n"); + Path fprPath = createMultiFileRemediationFpr("multi-file-fix", List.of( + new FileChangeSpec("Bad.java", 2, 2, 1, 1, "nomatch-a\nnomatch-b\nnomatch-c", "NOMATCH", "X"), + new FileChangeSpec("Good.java", 2, 2, 1, 1, "before\nTARGET\nafter", "TARGET", "REPLACED"))); + + RemediationMetric metric; + try (FprHandle fprHandle = new FprHandle(fprPath)) { + metric = new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + + assertEquals(1, metric.totalRemediations()); + assertEquals(0, metric.appliedRemediations()); + assertEquals(1, metric.skippedRemediations()); + assertEquals(Map.of("Source context not found", 1), metric.skippedByReason()); + assertEquals(Set.of(), metric.modifiedFiles()); + assertEquals("before\nTARGET\nafter\n", Files.readString(goodFile)); + assertEquals("one\ntwo\nthree\n", Files.readString(badFile)); + } + + // ------------------------------------------------------------------------------------------ + // Folded in from the former RemediationProcessorKnownIssuesTest (REVIEW_2 §2.1 - §2.5) once + // every scenario there turned green. Uses its own multi-hunk/multi-file harness below since + // the builders above only support one hunk per file. + // ------------------------------------------------------------------------------------------ + + /** + * A malformed remediation entry (missing {@code }) is skipped on its own via + * {@code REMEDIATION_DATA_INVALID} rather than aborting the whole batch; the well-formed + * remediation alongside it still applies. + */ + @Test + void malformedRemediationIsSkippedAndValidRemediationsStillApply() throws Exception { + Path sourceFile = writeSourceFile("Example.java", "before\nTARGET\nafter\n"); + Path fprPath = buildFpr(List.of( + new MultiHunkRemediationSpec("malformed", List.of(new FileSpec(null, List.of( + new HunkSpec(1, 1, 0, 0, "before", "before", "BROKEN"))))), + new MultiHunkRemediationSpec("valid", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(2, 2, 1, 1, "before\ntarget\nafter", "TARGET", "REPLACED"))))))); + + RemediationMetric metric = apply(fprPath); + + assertEquals(2, metric.totalRemediations()); + assertEquals(1, metric.appliedRemediations(), "the valid remediation must still be applied"); + assertEquals(1, metric.skippedRemediations()); + assertEquals(Map.of("Remediation data invalid", 1), metric.skippedByReason()); + assertEquals("before\nREPLACED\nafter\n", Files.readString(sourceFile)); + } + + /** + * Hunks staged earlier in the same remediation must be visible to the offset projection of a + * later hunk in the same file: hunk 1 replaces lines 2-3 with a single line (delta -1) and + * hunk 2 targets the second of two identical {@code before/TARGET/after} blocks. With the + * staged delta visible, the projected start lands exactly on one of the two context + * candidates instead of being ambiguous. + * + *

Note {@link #offsetLedgerAccountsForDedupWhenProjectingLaterHunkInSameFile} looks similar + * but does not cover this - it uses two separate remediations, so the first is committed + * before the second is classified. + */ + @Test + void secondHunkOfSameRemediationProjectsThroughFirstHunksLineDelta() throws Exception { + Path sourceFile = writeSourceFile("Example.java", + "line1\nline2\nline3\nbefore\nTARGET\nafter\nbefore\nTARGET\nafter\ntail\n"); + Path fprPath = buildFpr(List.of( + new MultiHunkRemediationSpec("two-hunks", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(2, 3, 1, 1, "line1\nline2\nline3\nbefore", "line2\nline3", "X"), + new HunkSpec(8, 8, 1, 1, "before\ntarget\nafter", "TARGET", "REPLACED"))))))); + + RemediationMetric metric = apply(fprPath); + + assertEquals(1, metric.appliedRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals(Map.of(), metric.skippedByReason()); + assertEquals("line1\nX\nbefore\nTARGET\nafter\nbefore\nREPLACED\nafter\ntail\n", + Files.readString(sourceFile)); + } + + /** + * A remediation mixing an already-covered hunk and a still-applicable hunk must apply the + * applicable one rather than being rejected wholesale: {@code wide-fix} covers lines 1-3, + * {@code mixed-fix}'s first hunk (line 2) is covered by it, but its second hunk (line 5) is + * untouched and must still land. + */ + @Test + void remediationMixingCoveredAndApplicableHunksStillAppliesTheApplicableOne() throws Exception { + Path sourceFile = writeSourceFile("Example.java", "before\nTARGET\nafter\nkeep\nOTHER\ntail\n"); + Path fprPath = buildFpr(List.of( + new MultiHunkRemediationSpec("wide-fix", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(1, 3, 0, 0, "before\nTARGET\nafter", "before\nTARGET\nafter", "W1\nW2\nW3"))))), + new MultiHunkRemediationSpec("mixed-fix", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(2, 2, 1, 1, "before\ntarget\nafter", "TARGET", "NARROW"), + new HunkSpec(5, 5, 1, 1, "keep\nOTHER\ntail", "OTHER", "FIXED"))))))); + + RemediationMetric metric = apply(fprPath); + + assertEquals(2, metric.totalRemediations()); + assertEquals(2, metric.appliedRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals("W1\nW2\nW3\nkeep\nFIXED\ntail\n", Files.readString(sourceFile)); + } + + /** + * A remediation with any conflicting hunk must not be applied at all, even if its other + * hunks would individually be POSSIBLY_REMEDIATED: {@code mixed-fix}'s hunk 1 (line 2) is + * POSSIBLY_REMEDIATED against {@code wide-a}, and hunk 2 (lines 7-8) CONFLICTS with + * {@code wide-b}'s declared range. The whole remediation must be skipped as conflicting, + * not partially applied. + */ + @Test + void remediationWithAnyConflictingHunkIsSkippedAsConflicting() throws Exception { + Path sourceFile = writeSourceFile("Example.java", "l1\nl2\nl3\nl4\nl5\nl6\nl7\nl8\n"); + Path fprPath = buildFpr(List.of( + new MultiHunkRemediationSpec("wide-a", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(1, 3, 0, 0, "l1\nl2\nl3", "l1\nl2\nl3", "A1\nl2\nA3"))))), + new MultiHunkRemediationSpec("wide-b", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(5, 7, 1, 1, "l4\nl5\nl6\nl7\nl8", "l5\nl6\nl7", "B5\nB6\nl7"))))), + new MultiHunkRemediationSpec("mixed-fix", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(2, 2, 1, 1, "l1\nl2\nl3", "l2", "M2"), + new HunkSpec(7, 8, 1, 1, "l6\nl7\nl8", "l7\nl8", "M7\nM8"))))))); + + RemediationMetric metric = apply(fprPath); + + assertEquals(3, metric.totalRemediations()); + assertEquals(2, metric.appliedRemediations()); + assertEquals(1, metric.skippedRemediations()); + assertEquals(Map.of("Conflicts with another fix", 1), metric.skippedByReason()); + assertEquals("A1\nl2\nA3\nl4\nB5\nB6\nl7\nl8\n", Files.readString(sourceFile), + "the conflicting hunk must not overwrite wide-b's region"); + } + + /** + * NewCode line endings must follow the file's own separator rather than an unrelated + * comment-marker-stripping side effect: for an extension unmapped by + * {@code FileTypeLanguageMapperUtil}, a CRLF NewCode (the CR here is a literal + * {@code } character reference) must not leave a stray CR in an LF-only source file. + */ + @Test + void newCodeWithCrlfDoesNotLeaveStrayCarriageReturnsInLfSourceFile() throws Exception { + Path sourceFile = writeSourceFile("payload.zzz", "before\nTARGET\nafter\n"); + Path fprPath = buildFpr(List.of( + new MultiHunkRemediationSpec("crlf-newcode", List.of(new FileSpec("payload.zzz", List.of( + new HunkSpec(2, 2, 1, 1, "before\ntarget\nafter", "TARGET", "NEW1 \nNEW2"))))))); + + RemediationMetric metric = apply(fprPath); + + assertEquals(1, metric.appliedRemediations()); + assertEquals("before\nNEW1\nNEW2\nafter\n", Files.readString(sourceFile), + "NewCode line endings must follow the file, not the XML payload"); + } + + /** + * {@code contentCovers} must not treat an incidental substring hit as proof of coverage: the + * broader fix's replacement happens to contain {@code return;} inside a null guard, while the + * narrower fix proposes {@code return;} at a different line - they are not the same fix, so + * the narrower remediation must classify as POSSIBLY_REMEDIATED, not SUPERSEDED. + */ + @Test + void incidentalSubstringMatchIsNotTreatedAsProvenSupersession() throws Exception { + Path sourceFile = writeSourceFile("Example.java", "value = getInput();\nprocess(value);\nfinish();\n"); + Path fprPath = buildFpr(List.of( + new MultiHunkRemediationSpec("wide-fix", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(1, 3, 0, 0, + "value = getInput();\nprocess(value);\nfinish();", + "value = getInput();\nprocess(value);\nfinish();", + "value = sanitize(getInput());\nif (value == null) { return; }\nprocess(value);"))))), + new MultiHunkRemediationSpec("narrow-fix", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(2, 2, 1, 1, "value = getInput();\nprocess(value);\nfinish();", + "process(value);", "return;"))))))); + + RemediationMetric metric = apply(fprPath); + + assertEquals(2, metric.totalRemediations()); + assertEquals(1, metric.appliedRemediations()); + assertEquals(0, metric.supersededRemediations(), + "an incidental substring hit is not proof that the broader fix covers this hunk"); + assertEquals(1, metric.possiblyRemediatedRemediations()); + assertEquals("value = sanitize(getInput());\nif (value == null) { return; }\nprocess(value);\n", + Files.readString(sourceFile), "only the broader fix may be written"); + } + + /** + * {@code contentCovers} must not treat a blank candidate comparison code as proof of + * coverage: a comment-only NewCode normalises to {@code ""}, and {@code anything.contains("")} + * is trivially true, so this must not be misread as SUPERSEDED. + */ + @Test + void blankCandidateComparisonCodeIsNotTreatedAsProvenSupersession() throws Exception { + Path sourceFile = writeSourceFile("Example.java", "before\nTARGET\nafter\n"); + Path fprPath = buildFpr(List.of( + new MultiHunkRemediationSpec("wide-fix", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(1, 3, 0, 0, "before\nTARGET\nafter", "before\nTARGET\nafter", + "W1\nW2\nW3"))))), + // NewCode is a single line comment, so its comparison code normalises to "". + new MultiHunkRemediationSpec("comment-only-fix", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(2, 2, 1, 1, "before\ntarget\nafter", "TARGET", + "// nothing to change here"))))))); + + RemediationMetric metric = apply(fprPath); + + assertEquals(2, metric.totalRemediations()); + assertEquals(1, metric.appliedRemediations()); + assertEquals(0, metric.supersededRemediations(), + "an empty candidate comparison code is not proof of coverage"); + assertEquals(1, metric.possiblyRemediatedRemediations()); + assertEquals("W1\nW2\nW3\n", Files.readString(sourceFile)); + } + + // ------------------------------------------------------------------------------------------ + // Folded in from the former RemediationProcessorRemainingIssuesTest (REVIEW_3 points 1-4) once + // every scenario there turned green. + // ------------------------------------------------------------------------------------------ + + /** + * A filename that is not a legal path on this OS ({@code FileChange.resolve} throwing + * {@code InvalidPathException}) must be translated to {@code REMEDIATION_DATA_INVALID} and + * skipped on its own, rather than escaping the per-remediation loop and aborting the whole + * batch; the well-formed remediation alongside it must still apply. + * + *

Windows only: {@code ':'} is illegal in a Windows path but legal on Linux, so the + * scenario cannot be reproduced there (the only character a Unix path rejects is NUL, which + * XML 1.0 cannot carry). + */ + @Test + @EnabledOnOs(OS.WINDOWS) + void illegalFilenameIsSkippedAndValidRemediationsStillApply() throws Exception { + Path sourceFile = writeSourceFile("Example.java", "before\nTARGET\nafter\n"); + Path fprPath = buildFpr(List.of( + new MultiHunkRemediationSpec("illegal-filename", List.of(new FileSpec("src/bad:name.java", List.of( + new HunkSpec(1, 1, 0, 0, "before", "before", "BROKEN"))))), + new MultiHunkRemediationSpec("valid", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(2, 2, 1, 1, "before\ntarget\nafter", "TARGET", "REPLACED"))))))); + + RemediationMetric metric = apply(fprPath); + + assertEquals(2, metric.totalRemediations()); + assertEquals(1, metric.appliedRemediations(), "the valid remediation must still be applied"); + assertEquals(1, metric.skippedRemediations()); + assertEquals(1, metric.skippedByReason().values().stream().mapToInt(Integer::intValue).sum(), + "the illegal filename must be recorded as exactly one skip"); + assertEquals("before\nREPLACED\nafter\n", Files.readString(sourceFile)); + } + + /** + * The offset ledger must record where a hunk actually landed via the fuzzy anchor, not where + * it was declared: {@code relocated} declares line 12 but its context/OriginalCode only exist + * at line 3, so it lands there with delta +2 and the ledger must stage {@code (3, 3, +2)} - + * not {@code (12, 12, +2)}. {@code later} genuinely targets the second {@code DUP} block at + * declared line 11, which needs that correctly-staged shift to disambiguate between the two + * identical context blocks; a stale staged range causes the shift to compute as 0 and the + * exact-position disambiguation to match neither candidate. + */ + @Test + void offsetLedgerRecordsWhereHunkActuallyLandedNotWhereItWasDeclared() throws Exception { + Path sourceFile = writeSourceFile("Example.java", + "aa\nHEAD\nTARGET\nTAIL\nbb\nctx\nDUP\nctx2\ncc\nctx\nDUP\nctx2\ndd\n"); + Path fprPath = buildFpr(List.of( + // Declared line 12 is stale; the anchor really sits at line 3. + new MultiHunkRemediationSpec("relocated", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(12, 12, 1, 1, "HEAD\nTARGET\nTAIL", "TARGET", "R1\nR2\nR3"))))), + // Genuinely targets the SECOND DUP block, declared (correctly) at line 11. + new MultiHunkRemediationSpec("later", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(11, 11, 1, 1, "ctx\nDUP\nctx2", "DUP", "FIXED"))))))); + + RemediationMetric metric = apply(fprPath); + + assertEquals(2, metric.appliedRemediations()); + assertEquals(0, metric.skippedRemediations()); + assertEquals(Map.of(), metric.skippedByReason()); + assertEquals("aa\nHEAD\nR1\nR2\nR3\nTAIL\nbb\nctx\nDUP\nctx2\ncc\nctx\nFIXED\nctx2\ndd\n", + Files.readString(sourceFile)); + } + + /** + * {@code contentCovers} must not treat an incidental substring hit as proof of coverage even + * above the minimum-length guard: {@code return null;} normalises to 11 characters and occurs + * in the wide fix's replacement only incidentally, inside a null guard several lines away from + * where the narrow fix targets - they are not the same fix, so this must classify as + * POSSIBLY_REMEDIATED, not SUPERSEDED. + */ + @Test + void incidentalSubstringAboveTheLengthGuardIsNotProvenSupersession() throws Exception { + Path sourceFile = writeSourceFile("Example.java", "value = getInput();\nprocess(value);\nfinish();\n"); + Path fprPath = buildFpr(List.of( + new MultiHunkRemediationSpec("wide-fix", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(1, 3, 0, 0, + "value = getInput();\nprocess(value);\nfinish();", + "value = getInput();\nprocess(value);\nfinish();", + "value = sanitize(getInput());\nif (value == null) { return null; }\nprocess(value);"))))), + new MultiHunkRemediationSpec("narrow-fix", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(2, 2, 1, 1, "value = getInput();\nprocess(value);\nfinish();", + "process(value);", "return null;"))))))); + + RemediationMetric metric = apply(fprPath); + + assertEquals(2, metric.totalRemediations()); + assertEquals(1, metric.appliedRemediations()); + assertEquals(0, metric.supersededRemediations(), + "an incidental substring hit is not proof that the broader fix covers this hunk, " + + "however long the candidate happens to be"); + assertEquals(1, metric.possiblyRemediatedRemediations()); + assertEquals("value = sanitize(getInput());\nif (value == null) { return null; }\nprocess(value);\n", + Files.readString(sourceFile), "only the broader fix may be written"); + } + + /** + * Safety property that per-FPR ledgers depend on: {@code apply-remediations --all-open-issues} + * uses a fresh {@code RemediationProcessor} (and therefore a fresh ledger) per artifact, since + * different artifacts may be scans of different source revisions. What keeps a multi-FPR run + * safe is not the ledger but anchor verification - once an earlier FPR has rewritten a file, + * the declared hash no longer matches, so a later FPR's hunk goes through the projection/fuzzy + * path and is written only where its OriginalCode still literally matches. Here the narrow + * fix's OriginalCode was consumed by the wide fix from the first FPR, so the second FPR must + * skip it rather than write it against stale text. + */ + @Test + void secondFprDoesNotApplyOverAFixTheFirstFprAlreadyRewrote() throws Exception { + Path sourceFile = writeSourceFile("Example.java", "value = getInput();\nprocess(value);\nfinish();\n"); + Path firstFpr = buildFpr("artifact-1.fpr", List.of( + new MultiHunkRemediationSpec("wide-fix", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(1, 3, 0, 0, + "value = getInput();\nprocess(value);\nfinish();", + "value = getInput();\nprocess(value);\nfinish();", + "value = sanitize(getInput());\nprocess(sanitized);\nfinish();"))))))); + Path secondFpr = buildFpr("artifact-2.fpr", List.of( + new MultiHunkRemediationSpec("narrow-fix", List.of(new FileSpec("Example.java", List.of( + new HunkSpec(2, 2, 1, 1, "value = getInput();\nprocess(value);\nfinish();", + "process(value);", "process(escape(value));"))))))); + + RemediationMetric first = apply(firstFpr); + assertEquals(1, first.appliedRemediations()); + String afterFirst = "value = sanitize(getInput());\nprocess(sanitized);\nfinish();\n"; + assertEquals(afterFirst, Files.readString(sourceFile)); + + RemediationMetric second = apply(secondFpr); + + assertEquals(0, second.appliedRemediations(), + "the narrower fix's OriginalCode no longer exists, so it must not be written anywhere"); + assertEquals(1, second.skippedRemediations()); + assertEquals(afterFirst, Files.readString(sourceFile), "the second FPR must leave the file untouched"); + } + + private record HunkSpec(int lineFrom, int lineTo, int contextBefore, int contextAfter, + String context, String originalCode, String newCode) {} + + /** A {@code } block; a {@code null} filename omits the element entirely. */ + private record FileSpec(String filename, List hunks) {} + + private record MultiHunkRemediationSpec(String instanceId, List files) {} + + private RemediationMetric apply(Path fprPath) throws Exception { + try (FprHandle fprHandle = new FprHandle(fprPath)) { + return new RemediationProcessor(fprHandle, tempDir.toString()).processRemediationXML(); + } + } + + private Path buildFpr(List specs) throws Exception { + return buildFpr("remediation.fpr", specs); + } + + private Path buildFpr(String fprName, List specs) throws Exception { + StringBuilder remediations = new StringBuilder(); + for (MultiHunkRemediationSpec spec : specs) { + remediations.append("\n") + .append(" test\n"); + for (FileSpec file : spec.files()) { + remediations.append(" \n"); + if (file.filename() != null) { + remediations.append(" ").append(file.filename()).append("\n"); + } + remediations.append(" not-the-source-hash\n"); + for (HunkSpec hunk : file.hunks()) { + remediations.append(""" + + %d + %d + %s + %s + %s + + """.formatted(hunk.lineFrom(), hunk.lineTo(), hunk.contextBefore(), + hunk.contextAfter(), hunk.context(), hunk.originalCode(), hunk.newCode())); + } + remediations.append(" \n"); + } + remediations.append("\n"); + } + String remediationXml = """ + + + + test + 2026-08-26T00:00:00Z + + + %s + + + """.formatted(REMEDIATIONS_NAMESPACE, remediations); + + Path fprPath = tempDir.resolve(fprName); + try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(fprPath))) { + zipOutputStream.putNextEntry(new ZipEntry("remediations.xml")); + zipOutputStream.write(remediationXml.getBytes(StandardCharsets.UTF_8)); + zipOutputStream.closeEntry(); + } + return fprPath; + } + + private static String sha256Base64(byte[] bytes) throws Exception { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return Base64.getEncoder().encodeToString(digest.digest(bytes)); + } + private Path writeSourceFile(String content) throws Exception { - Path sourceFile = tempDir.resolve("Example.java"); + return writeSourceFile("Example.java", content); + } + + private Path writeSourceFile(String filename, String content) throws Exception { + Path sourceFile = tempDir.resolve(filename); Files.writeString(sourceFile, content, StandardCharsets.UTF_8); return sourceFile; } @@ -105,6 +863,52 @@ private Path createRemediationFpr(String context, String originalCode, String ne return createRemediationFpr(2, 2, 1, 1, context, originalCode, newCode); } + private record RemediationSpec(String instanceId, int lineFrom, int lineTo, int contextBefore, int contextAfter, + String context, String originalCode, String newCode) {} + + private Path createRemediationFpr(List specs) throws Exception { + Path fprPath = tempDir.resolve("remediation.fpr"); + StringBuilder remediations = new StringBuilder(); + for (RemediationSpec spec : specs) { + remediations.append(""" + + test + + Example.java + not-the-source-hash + + %d + %d + %s + %s + %s + + + + """.formatted(spec.instanceId(), spec.lineFrom(), spec.lineTo(), spec.contextBefore(), + spec.contextAfter(), spec.context(), spec.originalCode(), spec.newCode())); + } + String remediationXml = """ + + + + test + 2026-08-26T00:00:00Z + + + %s + + + """.formatted(REMEDIATIONS_NAMESPACE, remediations); + + try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(fprPath))) { + zipOutputStream.putNextEntry(new ZipEntry("remediations.xml")); + zipOutputStream.write(remediationXml.getBytes(StandardCharsets.UTF_8)); + zipOutputStream.closeEntry(); + } + return fprPath; + } + private Path createRemediationFpr(int lineFrom, int lineTo, int contextBefore, int contextAfter, String context, String originalCode, String newCode) throws Exception { Path fprPath = tempDir.resolve("remediation.fpr"); @@ -142,4 +946,143 @@ private Path createRemediationFpr(int lineFrom, int lineTo, int contextBefore, i } return fprPath; } + + private Path createRemediationFprWithHash(int lineFrom, int lineTo, int contextBefore, int contextAfter, + String context, String originalCode, String newCode, String hash) throws Exception { + Path fprPath = tempDir.resolve("remediation.fpr"); + String remediationXml = """ + + + + test + 2026-08-26T00:00:00Z + + + + test + + Example.java + %s + + %d + %d + %s + %s + %s + + + + + + """.formatted(REMEDIATIONS_NAMESPACE, hash, lineFrom, lineTo, contextBefore, contextAfter, + context, originalCode, newCode); + + try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(fprPath))) { + zipOutputStream.putNextEntry(new ZipEntry("remediations.xml")); + zipOutputStream.write(remediationXml.getBytes(StandardCharsets.UTF_8)); + zipOutputStream.closeEntry(); + } + return fprPath; + } + + private record FileChangeSpec(String filename, int lineFrom, int lineTo, int contextBefore, int contextAfter, + String context, String originalCode, String newCode) {} + + private Path createMultiFileRemediationFpr(String instanceId, List fileChangeSpecs) throws Exception { + Path fprPath = tempDir.resolve("remediation.fpr"); + StringBuilder fileChanges = new StringBuilder(); + for (FileChangeSpec spec : fileChangeSpecs) { + fileChanges.append(""" + + %s + not-the-source-hash + + %d + %d + %s + %s + %s + + + """.formatted(spec.filename(), spec.lineFrom(), spec.lineTo(), spec.contextBefore(), + spec.contextAfter(), spec.context(), spec.originalCode(), spec.newCode())); + } + String remediationXml = """ + + + + test + 2026-08-26T00:00:00Z + + + + test + %s + + + + """.formatted(REMEDIATIONS_NAMESPACE, instanceId, fileChanges); + + try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(fprPath))) { + zipOutputStream.putNextEntry(new ZipEntry("remediations.xml")); + zipOutputStream.write(remediationXml.getBytes(StandardCharsets.UTF_8)); + zipOutputStream.closeEntry(); + } + return fprPath; + } + + /** + * Builds an FPR with multiple remediations (insertion order preserved), each of which may + * itself contain multiple file changes/hunks - used to construct mixed-outcome scenarios + * where hunks within the same remediation classify differently against the ledger. + */ + private Path createRemediationFprWithRemediations(LinkedHashMap> remediationFileChanges) + throws Exception { + Path fprPath = tempDir.resolve("remediation.fpr"); + StringBuilder remediationsXml = new StringBuilder(); + for (Map.Entry> entry : remediationFileChanges.entrySet()) { + StringBuilder fileChanges = new StringBuilder(); + for (FileChangeSpec spec : entry.getValue()) { + fileChanges.append(""" + + %s + not-the-source-hash + + %d + %d + %s + %s + %s + + + """.formatted(spec.filename(), spec.lineFrom(), spec.lineTo(), spec.contextBefore(), + spec.contextAfter(), spec.context(), spec.originalCode(), spec.newCode())); + } + remediationsXml.append(""" + + test + %s + + """.formatted(entry.getKey(), fileChanges)); + } + String remediationXml = """ + + + + test + 2026-08-26T00:00:00Z + + + %s + + + """.formatted(REMEDIATIONS_NAMESPACE, remediationsXml); + + try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(fprPath))) { + zipOutputStream.putNextEntry(new ZipEntry("remediations.xml")); + zipOutputStream.write(remediationXml.getBytes(StandardCharsets.UTF_8)); + zipOutputStream.closeEntry(); + } + return fprPath; + } } diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/remediation/model/AppliedChangeTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/remediation/model/AppliedChangeTest.java new file mode 100644 index 00000000000..cc902959a23 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/remediation/model/AppliedChangeTest.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.remediation.model; + +import static org.junit.jupiter.api.Assertions.assertFalse; + +import org.junit.jupiter.api.Test; + +class AppliedChangeTest { + + /** + * {@code contentCovers} must return {@code false}, not {@code true}, when either side's + * comparison code is unavailable ({@code null}) or blank - unavailable content is not proof + * of coverage. These branches are unreachable end-to-end through + * {@code RemediationProcessor.processRemediationXML} + * (see {@link RemediationProcessorKnownIssuesTest#malformedRemediationIsSkippedAndValidRemediationsStillApply}: + * a missing comparison code causes the remediation to be skipped before the classifier is + * ever reached for it), so this is asserted directly against {@link AppliedChange}. + */ + @Test + void unavailableComparisonCodeIsNotTreatedAsProvenCoverage() { + assertFalse(new AppliedChange(1, 3, 0, "W1W2W3").contentCovers(null, 2, 2), + "a candidate whose content could not be computed has not been proven covered"); + assertFalse(new AppliedChange(1, 3, 0, null).contentCovers("M2", 2, 2), + "an applied change whose content is unknown cannot prove it covers anything"); + } +} diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/util/FileUtilTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/util/FileUtilTest.java new file mode 100644 index 00000000000..ef6f959fe7d --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/util/FileUtilTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Map; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import com.fortify.cli.aviator.config.LanguagesCommentConfig; + +class FileUtilTest { + + @BeforeAll + static void initializeCommentConfig() { + LanguagesCommentConfig commentConfig = new LanguagesCommentConfig(); + commentConfig.setLineCommentSymbols(Map.of( + "HTML", "