Skip to content

WIP(sqlInteceptor): support external sql interceptor call back #4488

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: dev/4.3.4
Choose a base branch
from
Open

WIP(sqlInteceptor): support external sql interceptor call back #4488

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 25 additions & 3 deletions ...c-service/src/main/java/com/oceanbase/odc/service/integration/ExternalSqlInterceptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package com.oceanbase.odc.service.integration;

import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
Expand All @@ -25,6 +26,7 @@
import org.springframework.stereotype.Component;

import com.oceanbase.odc.common.json.JsonUtils;
import com.oceanbase.odc.common.util.MapUtils;
import com.oceanbase.odc.core.session.ConnectionSession;
import com.oceanbase.odc.core.session.ConnectionSessionUtil;
import com.oceanbase.odc.core.shared.constant.OrganizationType;
Expand All @@ -37,8 +39,9 @@
import com.oceanbase.odc.service.connection.model.ConnectionConfig;
import com.oceanbase.odc.service.iam.auth.AuthenticationFacade;
import com.oceanbase.odc.service.integration.client.SqlInterceptorClient;
import com.oceanbase.odc.service.integration.model.SqlCheckStatus;
import com.oceanbase.odc.service.integration.model.SqlCheckResult;
import com.oceanbase.odc.service.integration.model.SqlInterceptorProperties;
import com.oceanbase.odc.service.integration.model.SqlInterceptorProperties.CheckProperties;
import com.oceanbase.odc.service.integration.model.TemplateVariables;
import com.oceanbase.odc.service.integration.model.TemplateVariables.Variable;
import com.oceanbase.odc.service.regulation.ruleset.RuleService;
Expand Down Expand Up @@ -100,9 +103,20 @@ public boolean doPreHandle(@NonNull SqlAsyncExecuteReq request, @NonNull SqlAsyn
SqlInterceptorProperties properties =
(SqlInterceptorProperties) integrationService.getIntegrationProperties(interceptorOpt.get().getId());
TemplateVariables variables = buildTemplateVariables(request.getSql(), session);
SqlCheckStatus result = sqlInterceptorClient.check(properties, variables);
switch (result) {
SqlCheckResult result = sqlInterceptorClient.check(properties, variables);
// set variables in template
if (!MapUtils.isEmpty(result.getExtractedResponse())) {
for (Map.Entry<String, String> entry : result.getExtractedResponse().entrySet()) {
variables.setAttribute(Variable.EXTERNAL_PROPERTIES, entry.getKey(), entry.getValue());
}
}
CheckProperties check = properties.getApi().getCheck();
switch (result.getSqlCheckStatus()) {
case IN_WHITE_LIST:
if (null != check.onInWhiteList()) {
sqlInterceptorClient.getIntegrationResponse(properties.getHttp(), check.onInWhiteList(), variables,
properties.getEncryption());
}
return true;
case IN_BLACK_LIST:
ruleService.getByRulesetIdAndName(ruleSetId, SqlConsoleRules.EXTERNAL_SQL_INTERCEPTOR.getRuleName())
Expand All @@ -121,6 +135,10 @@ public boolean doPreHandle(@NonNull SqlAsyncExecuteReq request, @NonNull SqlAsyn
violationRule.setViolation(violation);
response.getViolatedRules().add(violationRule);
});
if (null != check.onInBlackList()) {
sqlInterceptorClient.getIntegrationResponse(properties.getHttp(), check.onInBlackList(), variables,
properties.getEncryption());
}
return false;
case NEED_REVIEW:
ruleService.getByRulesetIdAndName(ruleSetId, SqlConsoleRules.EXTERNAL_SQL_INTERCEPTOR.getRuleName())
Expand All @@ -139,6 +157,10 @@ public boolean doPreHandle(@NonNull SqlAsyncExecuteReq request, @NonNull SqlAsyn
violationRule.setViolation(violation);
response.getViolatedRules().add(violationRule);
});
if (null != check.onNeedReview()) {
sqlInterceptorClient.getIntegrationResponse(properties.getHttp(), check.onNeedReview(), variables,
properties.getEncryption());
}
return false;
default:
throw new UnexpectedException("SQL intercept failed, unknown intercept status: " + result);
Expand Down
69 changes: 51 additions & 18 deletions ...vice/src/main/java/com/oceanbase/odc/service/integration/client/SqlInterceptorClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
*/
package com.oceanbase.odc.service.integration.client;

import java.util.HashMap;
import java.util.Map;

import javax.annotation.PostConstruct;

import org.apache.http.client.HttpClient;
Expand All @@ -25,17 +28,22 @@
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import com.oceanbase.odc.common.util.MapUtils;
import com.oceanbase.odc.core.shared.Verify;
import com.oceanbase.odc.core.shared.constant.ErrorCodes;
import com.oceanbase.odc.core.shared.exception.ExternalServiceError;
import com.oceanbase.odc.core.shared.exception.UnexpectedException;
import com.oceanbase.odc.service.integration.HttpOperationService;
import com.oceanbase.odc.service.integration.model.ApprovalProperties;
import com.oceanbase.odc.service.integration.model.Encryption;
import com.oceanbase.odc.service.integration.model.IntegrationProperties;
import com.oceanbase.odc.service.integration.model.IntegrationProperties.ApiProperties;
import com.oceanbase.odc.service.integration.model.IntegrationProperties.HttpProperties;
import com.oceanbase.odc.service.integration.model.OdcIntegrationResponse;
import com.oceanbase.odc.service.integration.model.SqlCheckResult;
import com.oceanbase.odc.service.integration.model.SqlCheckStatus;
import com.oceanbase.odc.service.integration.model.SqlInterceptorProperties;
import com.oceanbase.odc.service.integration.model.SqlInterceptorProperties.CallBackProperties;
import com.oceanbase.odc.service.integration.model.SqlInterceptorProperties.CheckProperties;
import com.oceanbase.odc.service.integration.model.TemplateVariables;
import com.oceanbase.odc.service.integration.model.TemplateVariables.Variable;
Expand Down Expand Up @@ -82,40 +90,65 @@ public void init() {
* @param variables Template variables for building request, more details reference {@link Variable}
* @return The check result {@link SqlCheckStatus} of SQL content
*/
public SqlCheckStatus check(@NonNull SqlInterceptorProperties properties, TemplateVariables variables) {
public SqlCheckResult check(@NonNull SqlInterceptorProperties properties, TemplateVariables variables) {
CheckProperties check = properties.getApi().getCheck();
HttpProperties http = properties.getHttp();
Encryption encryption = properties.getEncryption();
HttpUriRequest request;
try {
request = httpService.buildHttpRequest(check, http, encryption, variables);
} catch (Exception e) {
throw new UnexpectedException("Build request failed: " + e.getMessage());
}
OdcIntegrationResponse response;
try {
response = httpClient.execute(request, new OdcIntegrationResponseHandler());
} catch (Exception e) {
throw new ExternalServiceError(ErrorCodes.ExternalServiceError,
"Request execute failed: " + e.getMessage());
}
response.setContent(EncryptionUtil.decrypt(response.getContent(), encryption));
OdcIntegrationResponse response = getIntegrationResponse(http, check, variables, encryption);
try {
SqlCheckStatus sqlCheckStatus = null;
String expression = check.getRequestSuccessExpression();
boolean valid = httpService.extractHttpResponse(response, expression, Boolean.class);
Verify.verify(valid, "Response is invalid, except: " + expression + ", response body: " + response);
if (httpService.extractHttpResponse(response, check.getInWhiteListExpression(), Boolean.class)) {
return SqlCheckStatus.IN_WHITE_LIST;
sqlCheckStatus = SqlCheckStatus.IN_WHITE_LIST;
} else if (httpService.extractHttpResponse(response, check.getInBlackListExpression(), Boolean.class)) {
return SqlCheckStatus.IN_BLACK_LIST;
sqlCheckStatus = SqlCheckStatus.IN_BLACK_LIST;
} else if (httpService.extractHttpResponse(response, check.getNeedReviewExpression(), Boolean.class)) {
return SqlCheckStatus.NEED_REVIEW;
sqlCheckStatus = SqlCheckStatus.NEED_REVIEW;
} else {
throw new RuntimeException(
"Response mismatch any check result expression, response body: " + response.getContent());
}
// try extract value from response for future request
Map<String, String> extractedResponse = new HashMap<>();
CallBackProperties callBackProperties = check.getCallback();
if (null != callBackProperties && !MapUtils.isEmpty(callBackProperties.getResponseExtractExpressions())) {
for (Map.Entry<String, String> responseExtractExpressionEntrySet : callBackProperties
.getResponseExtractExpressions().entrySet()) {
String key = responseExtractExpressionEntrySet.getKey();
String responseExtractExpression = responseExtractExpressionEntrySet.getValue();
String value = extractedResponse.put(key,
httpService.extractHttpResponse(response, responseExtractExpression, String.class));
if (null != value) {
extractedResponse.put(key, value);
}
}
}
return new SqlCheckResult(extractedResponse, sqlCheckStatus);
} catch (Exception e) {
throw new UnexpectedException("Extract SQL check result failed: " + e.getMessage());
}
}

public OdcIntegrationResponse getIntegrationResponse(HttpProperties http,
@NonNull IntegrationProperties.ApiProperties api, TemplateVariables variables, Encryption encryption) {
HttpUriRequest request;
try {
request = httpService.buildHttpRequest(api, http, encryption, variables);
} catch (Exception e) {
throw new UnexpectedException("Build request failed: " + e.getMessage());
}
OdcIntegrationResponse response;
try {
response = httpClient.execute(request, new OdcIntegrationResponseHandler());
} catch (Exception e) {
throw new ExternalServiceError(ErrorCodes.ExternalServiceError,
"Request execute failed: " + e.getMessage());
}
response.setContent(EncryptionUtil.decrypt(response.getContent(), encryption));
log.info("sqlInterceptorClient getIntegrationResponse request = {}, response ={}", request,
response.getContent());
return response;
}
}
32 changes: 32 additions & 0 deletions ...odc-service/src/main/java/com/oceanbase/odc/service/integration/model/SqlCheckResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/*
* Copyright (c) 2023 OceanBase.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.oceanbase.odc.service.integration.model;

import java.util.Map;

import lombok.AllArgsConstructor;
import lombok.Data;

/**
* @author longpeng.zlp
* @date 2025/6/19 13:49
*/
@Data
@AllArgsConstructor
public class SqlCheckResult {
private Map<String, String> extractedResponse;
private SqlCheckStatus sqlCheckStatus;
}
36 changes: 36 additions & 0 deletions ...e/src/main/java/com/oceanbase/odc/service/integration/model/SqlInterceptorProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
*/
package com.oceanbase.odc.service.integration.model;

import java.util.Map;

import javax.validation.Valid;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
Expand Down Expand Up @@ -51,6 +53,40 @@ public static class CheckProperties extends ApiProperties {
private String inBlackListExpression;
@NotBlank
private String needReviewExpression;
// call back properties
private CallBackProperties callback;

public ApiProperties onNeedReview() {
if (null != callback) {
return callback.onNeedReview;
} else {
return null;
}
}

public ApiProperties onInBlackList() {
if (null != callback) {
return callback.onInBlackList;
} else {
return null;
}
}

public ApiProperties onInWhiteList() {
if (null != callback) {
return callback.onInWhiteList;
} else {
return null;
}
}
}

@Data
public static class CallBackProperties {
private ApiProperties onNeedReview;
private ApiProperties onInWhiteList;
private ApiProperties onInBlackList;
private Map<String, String> responseExtractExpressions;
}

public static SqlInterceptorProperties from(IntegrationConfig config) {
Expand Down
3 changes: 2 additions & 1 deletion ...-service/src/main/java/com/oceanbase/odc/service/integration/model/TemplateVariables.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ public enum Variable {
PROJECT_OWNER_IDS("project.owner.ids"),
PROJECT_OWNER_ACCOUNTS("project.owner.accounts"),
PROJECT_OWNER_NAMES("project.owner.names"),
ODC_TASK_URL("odc.task.url");
ODC_TASK_URL("odc.task.url"),
EXTERNAL_PROPERTIES("external.response");


private final String key;
Expand Down
Loading