diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java index 726b8180dfc..cda333482d3 100755 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java @@ -49,7 +49,7 @@ public class EmbeddedServiceDefsUtil { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedServiceDefsUtil.class); // following servicedef list should be reviewed/updated whenever a new embedded service-def is added - public static final String DEFAULT_BOOTSTRAP_SERVICEDEF_LIST = "tag,gds,hdfs,hbase,hive,kms,knox,storm,yarn,kafka,solr,atlas,nifi,nifi-registry,sqoop,kylin,elasticsearch,presto,trino,ozone,kudu,schema-registry,nestedstructure,polaris"; + public static final String DEFAULT_BOOTSTRAP_SERVICEDEF_LIST = "tag,gds,hdfs,hbase,hive,kms,knox,storm,yarn,kafka,solr,atlas,nifi,nifi-registry,sqoop,kylin,elasticsearch,presto,trino,ozone,kudu,schema-registry,nestedstructure,polaris,s3"; public static final String EMBEDDED_SERVICEDEF_TAG_NAME = "tag"; public static final String EMBEDDED_SERVICEDEF_GDS_NAME = "gds"; public static final String EMBEDDED_SERVICEDEF_HDFS_NAME = "hdfs"; @@ -76,6 +76,7 @@ public class EmbeddedServiceDefsUtil { public static final String EMBEDDED_SERVICEDEF_KUDU_NAME = "kudu"; public static final String EMBEDDED_SERVICEDEF_NESTEDSTRUCTURE_NAME = "nestedstructure"; public static final String EMBEDDED_SERVICEDEF_POLARIS_NAME = "polaris"; + public static final String EMBEDDED_SERVICEDEF_S3_NAME = "s3"; public static final String PROPERTY_CREATE_EMBEDDED_SERVICE_DEFS = "ranger.service.store.create.embedded.service-defs"; public static final String HDFS_IMPL_CLASS_NAME = "org.apache.ranger.services.hdfs.RangerServiceHdfs"; @@ -98,7 +99,7 @@ public class EmbeddedServiceDefsUtil { private static final String PROPERTY_SUPPORTED_SERVICE_DEFS = "ranger.supportedcomponents"; private static final EmbeddedServiceDefsUtil instance = new EmbeddedServiceDefsUtil(); - private final RangerAdminConfig config; + private final RangerAdminConfig config; private Set supportedServiceDefs; private boolean createEmbeddedServiceDefs = true; private RangerServiceDef hdfsServiceDef; @@ -125,6 +126,7 @@ public class EmbeddedServiceDefsUtil { private RangerServiceDef kuduServiceDef; private RangerServiceDef nestedStructureServiveDef; private RangerServiceDef polarisServiceDef; + private RangerServiceDef s3ServiceDef; private RangerServiceDef tagServiceDef; private RangerServiceDef gdsServiceDef; @@ -189,6 +191,7 @@ public void init(ServiceStore store) { kuduServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KUDU_NAME); nestedStructureServiveDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_NESTEDSTRUCTURE_NAME); polarisServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_POLARIS_NAME); + s3ServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_S3_NAME); gdsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_GDS_NAME); @@ -296,6 +299,10 @@ public long getPolarisServiceDefId() { return getId(polarisServiceDef); } + public long getS3ServiceDefId() { + return getId(s3ServiceDef); + } + public long getTagServiceDefId() { return getId(tagServiceDef); } diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-s3.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-s3.json new file mode 100644 index 00000000000..36a6ff37c2a --- /dev/null +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-s3.json @@ -0,0 +1,144 @@ +{ + "id":21, + "name": "s3", + "displayName": "S3", + "implClass": "org.apache.ranger.services.s3.RangerServiceS3", + "label": "S3 Repository", + "description": "S3 Repository", + "guid": "", + "resources": + [ + { + "itemId": 1, + "name": "path", + "type": "path", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { "wildCard":true, "ignoreCase":false }, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "S3 Bucket/Object Path", + "description": "S3 Bucket or object path" + } + ], + + "accessTypes": + [ + { + "itemId": 1, + "name": "s3:ListBucket", + "label": "ListBucket" + }, + + { + "itemId": 2, + "name": "s3:PutObject", + "label": "PutObject" + }, + + { + "itemId": 3, + "name": "s3:GetObject", + "label": "GetObject" + }, + + { + "itemId": 4, + "name": "s3:DeleteObject", + "label": "DeleteObject" + } + ], + + "configs": + [ + { + "itemId": 1, + "name": "username", + "type": "string", + "subType": "", + "mandatory": true, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Username" + }, + { + "itemId": 2, + "name": "accesskey", + "type": "string", + "subType": "", + "mandatory": true, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Access key" + }, + + { + "itemId": 3, + "name": "secretkey", + "type": "password", + "subType": "", + "mandatory": true, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Secret key" + }, + + { + "itemId": 4, + "name": "endpoint", + "type": "string", + "subType": "", + "mandatory": true, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "End Point URL" + }, + { + "itemId": 5, + "name": "region", + "type": "string", + "subType": "", + "mandatory": true, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Region" + }, + + { + "itemId": 6, + "name": "bucketname", + "type": "string", + "subType": "", + "mandatory": true, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Bucket Name" + } + ], + + "enums": + [ + ], + + "contextEnrichers": + [ + + ], + + "policyConditions": + [ + + ] +} \ No newline at end of file diff --git a/distro/pom.xml b/distro/pom.xml index fa97c3fbd12..425ffbc1692 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -71,6 +71,13 @@ ${project.version} provided + + org.apache.ranger + pamCredValidator + ${project.version} + uexe + provided + org.apache.ranger ranger-atlas-plugin diff --git a/distro/src/main/assembly/admin-web.xml b/distro/src/main/assembly/admin-web.xml index b46b6244687..0ee26956a60 100644 --- a/distro/src/main/assembly/admin-web.xml +++ b/distro/src/main/assembly/admin-web.xml @@ -516,6 +516,26 @@ + + + true + + org.apache.ranger:s3-agent + + + ews/webapp/WEB-INF/classes/ranger-plugins/s3 + true + false + 755 + 644 + + software.amazon.awssdk:s3 + software.amazon.awssdk:auth + software.amazon.awssdk:regions + software.amazon.awssdk:iam + + + diff --git a/pom.xml b/pom.xml index 61140d18024..1d8d2509461 100644 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,7 @@ 1.3.7 2.4.0 1.12.765 + 2.17.102 1.84 2.2.0-b23 true @@ -970,6 +971,7 @@ ranger-tools ranger-util ranger-yarn-plugin-shim + s3-agent security-admin storm-agent tagsync @@ -995,6 +997,7 @@ agents-common common-utils + s3-agent security-admin ugsync-util @@ -1321,6 +1324,7 @@ ranger-tools ranger-util ranger-yarn-plugin-shim + s3-agent security-admin storm-agent tagsync @@ -1409,6 +1413,7 @@ ranger-tools ranger-util ranger-yarn-plugin-shim + s3-agent security-admin storm-agent tagsync diff --git a/s3-agent/pom.xml b/s3-agent/pom.xml new file mode 100644 index 00000000000..155d8d69d81 --- /dev/null +++ b/s3-agent/pom.xml @@ -0,0 +1,66 @@ + + + + 4.0.0 + + org.apache.ranger + ranger + 3.0.0-SNAPSHOT + .. + + s3-agent + jar + S3 Policies Agent + S3 Policies Agent + + UTF-8 + + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.databind.version} + + + org.apache.ranger + ranger-plugins-common + ${project.version} + + + software.amazon.awssdk + auth + ${aws-java-sdk2.version} + + + software.amazon.awssdk + iam + ${aws-java-sdk2.version} + + + software.amazon.awssdk + regions + ${aws-java-sdk2.version} + + + software.amazon.awssdk + s3 + ${aws-java-sdk2.version} + + + + diff --git a/s3-agent/src/main/java/org/apache/ranger/services/s3/RangerS3Constants.java b/s3-agent/src/main/java/org/apache/ranger/services/s3/RangerS3Constants.java new file mode 100644 index 00000000000..ff59dbd692c --- /dev/null +++ b/s3-agent/src/main/java/org/apache/ranger/services/s3/RangerS3Constants.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.s3; + +public class RangerS3Constants { + public static final String S3 = "s3"; + public static final String AWS = "AWS"; + public static final String ALLOW = "Allow"; + public static final String DENY = "Deny"; + public static final String S3_POLICY_LANGUAGE_VERSION = "2012-10-17"; + public static final String USER_NAME = "username"; + public static final String ACCESS_KEY = "accesskey"; + public static final String SECRET_KEY = "secretkey"; + public static final String ENDPOINT = "endpoint"; + public static final String REGION = "region"; + public static final String BUCKET_NAME = "bucketname"; + public static final String PATH = "path"; + public static final String S3_RESOURCE_PATH_ARN = "arn:aws:s3:::"; + public static final String S3_AWS_ACCOUNT_URN = "arn:aws:iam::"; + public static final int MAX_AUTOCOMPLETE_RESULTS = 100; + public static final int S3_LIST_OBJECTS_MAX_KEYS = 500; + + private RangerS3Constants() { + } +} diff --git a/s3-agent/src/main/java/org/apache/ranger/services/s3/RangerServiceS3.java b/s3-agent/src/main/java/org/apache/ranger/services/s3/RangerServiceS3.java new file mode 100644 index 00000000000..28d42fb802c --- /dev/null +++ b/s3-agent/src/main/java/org/apache/ranger/services/s3/RangerServiceS3.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.services.s3; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.service.RangerBaseService; +import org.apache.ranger.plugin.service.ResourceLookupContext; +import org.apache.ranger.services.s3.client.S3ResourceMgr; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.s3.model.S3Exception; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RangerServiceS3 extends RangerBaseService { + private static final Logger LOG = LoggerFactory.getLogger(RangerServiceS3.class); + + public static final String ACCESS_TYPE_LIST_BUCKET = "s3:ListBucket"; + + public RangerServiceS3() { + super(); + } + + @Override + public void init(RangerServiceDef serviceDef, RangerService service) { + super.init(serviceDef, service); + } + + @Override + public Map validateConfig() throws Exception { + Map ret = new HashMap(); + String serviceName = getServiceName(); + LOG.info("==> RangerServiceS3.validateConfig Service: (" + serviceName + " )"); + + if (configs != null) { + try { + ret = S3ResourceMgr.connectionTest(serviceName, configs); + } catch (S3Exception e) { + LOG.error("<== RangerServiceS3.validateConfig Error: " + e.getMessage(), e); + throw e; + } + } + + LOG.info("<== RangerServiceS3.validateConfig Response : (" + ret + " )"); + + return ret; + } + + @Override + public List lookupResource(ResourceLookupContext context) throws Exception { + List ret = new ArrayList(); + String serviceName = getServiceName(); + Map configs = getConfigs(); + + LOG.info("==> RangerServiceS3.lookupResource Context: (" + context + ")"); + + if (context != null) { + try { + long timeLookup = Long.parseLong(this.config.getProperties().getProperty("ranger.resource.lookup.timeout.value.in.ms")); + ret = S3ResourceMgr.getS3Resources(serviceName, configs, context, timeLookup); + } catch (S3Exception e) { + LOG.error("<==RangerServiceS3.lookupResource Error : " + e); + throw e; + } + } + + LOG.info("<== RangerServiceS3.lookupResource Response Received"); + + return ret; + } + + @Override + public List getDefaultRangerPolicies() throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceS3.getDefaultRangerPolicies()"); + } + + List ret = super.getDefaultRangerPolicies(); + + for (RangerPolicy defaultPolicy : ret) { + String lookUpUser = configs.get(RangerS3Constants.USER_NAME); + String bucketName = configs.get(RangerS3Constants.BUCKET_NAME); + + if (defaultPolicy.getName().contains("all") && StringUtils.isNotBlank(lookUpUser)) { + // Update the resource path value to use bucket name instead of wildcard + RangerPolicy.RangerPolicyResource pathPolicyResource = defaultPolicy.getResources().get(RangerS3Constants.PATH); + if (pathPolicyResource != null) { + // Clear existing values and set bucket name + pathPolicyResource.setValues(Collections.singletonList(bucketName)); + } + + // Clear existing policy items and create a new one with only ListBucket access + defaultPolicy.getPolicyItems().clear(); + + RangerPolicy.RangerPolicyItem policyItemForLookupUser = new RangerPolicy.RangerPolicyItem(); + + // Set only ListBucket access + List accessListForLookupUser = new ArrayList<>(); + accessListForLookupUser.add(new RangerPolicy.RangerPolicyItemAccess(ACCESS_TYPE_LIST_BUCKET, true)); + policyItemForLookupUser.setAccesses(accessListForLookupUser); + + // Set the lookup user + policyItemForLookupUser.setUsers(Collections.singletonList(lookUpUser)); + + // Set delegate admin to true (or false based on your requirement) + policyItemForLookupUser.setDelegateAdmin(true); + + // Add the modified policy item + defaultPolicy.addPolicyItem(policyItemForLookupUser); + + LOG.info("Modified default policy for S3 service: policy=" + defaultPolicy.getName() + + ", bucketName=" + bucketName + ", user=" + lookUpUser + ", access=ListBucket"); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceS3.getDefaultRangerPolicies()"); + } + + return ret; + } +} diff --git a/s3-agent/src/main/java/org/apache/ranger/services/s3/client/S3ClientConnectionMgr.java b/s3-agent/src/main/java/org/apache/ranger/services/s3/client/S3ClientConnectionMgr.java new file mode 100644 index 00000000000..3fbdee577db --- /dev/null +++ b/s3-agent/src/main/java/org/apache/ranger/services/s3/client/S3ClientConnectionMgr.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.services.s3.client; + +import org.apache.ranger.plugin.client.BaseClient; +import org.apache.ranger.services.s3.RangerS3Constants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.iam.IamClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.S3Exception; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +public class S3ClientConnectionMgr extends BaseClient { + private static final Logger LOG = LoggerFactory.getLogger(S3ClientConnectionMgr.class); + + public S3ClientConnectionMgr(String svcName, Map connectionProperties) { + super(svcName, connectionProperties); + } + + public static Map connectionTest(String serviceName, Map configs) { + LOG.debug("==> S3ClientConnectionMgr.connectionTest ServiceName: " + serviceName + "Configs" + configs); + boolean connectivityStatus = false; + Map responseData = new HashMap(); + // String bucketName = "odp-ranger-test"; + S3Client s3 = getS3client(configs); + String bucketName = configs.get(RangerS3Constants.BUCKET_NAME); + + try { + // Using ListObjectsV2 approach + ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder() + .bucket(bucketName) + .maxKeys(1) // Just need to check if bucket is accessible + .build(); + ListObjectsV2Response listObjectsResponse = s3.listObjectsV2(listObjectsRequest); + + // If we get here without exception, bucket exists and is accessible + connectivityStatus = true; + String successMsg = "ConnectionTest Successful - Bucket '" + bucketName + "' is accessible"; + generateResponseDataMap(connectivityStatus, successMsg, successMsg, null, null, responseData); + } catch (NoSuchBucketException e) { + String failureMsg = "Bucket '" + bucketName + "' does not exist"; + generateResponseDataMap(connectivityStatus, failureMsg, failureMsg, null, null, responseData); + LOG.error("<== S3ClientConnectionMgr.testConnection Error: " + e.getMessage(), e); + } catch (S3Exception e) { + String failureMsg = "Unable to connect to S3 using given parameters: " + e.awsErrorDetails().errorMessage(); + generateResponseDataMap(connectivityStatus, failureMsg, failureMsg, null, null, responseData); + LOG.error("<== S3ClientConnectionMgr.testConnection Error: " + e.getMessage(), e); + } catch (Exception e) { + String failureMsg = "Unexpected error during connection test: " + e.getMessage(); + generateResponseDataMap(connectivityStatus, failureMsg, failureMsg, null, null, responseData); + LOG.error("<== S3ClientConnectionMgr.testConnection Error: " + e.getMessage(), e); + } + + /*ListBucketsRequest listBucketsRequest = ListBucketsRequest.builder().build(); + ListBucketsResponse listBucketsResponse = s3.listBuckets(listBucketsRequest); + + if (listBucketsResponse != null) { + connectivityStatus = true; + String successMsg = "ConnectionTest Successful"; + generateResponseDataMap(connectivityStatus, successMsg, successMsg, + null, null, responseData); + } else { + String failureMsg = "Unable to connect to S3 using given parameters."; + generateResponseDataMap(connectivityStatus, failureMsg, failureMsg, + null, null, responseData); + } + } catch (S3Exception e) { + String failureMsg = "Unable to connect to S3 using given parameters."; + generateResponseDataMap(connectivityStatus, failureMsg, failureMsg, + null, null, responseData); + LOG.error("<== S3ClientConnectionMgr.testConnection Error: " + e.getMessage(), e); + } */ + + LOG.debug("<== S3ClientConnectionMgr.connectionTest Result : " + responseData); + return responseData; + } + + public static S3Client getS3client(Map configs) { + String accessKey = configs.get(RangerS3Constants.ACCESS_KEY); + String secretKey = configs.get(RangerS3Constants.SECRET_KEY); + String endPointOCE = configs.get(RangerS3Constants.ENDPOINT); + String regionstr = configs.get(RangerS3Constants.REGION); + AwsBasicCredentials awsCreds3 = AwsBasicCredentials.create(accessKey, secretKey); + Region region = Region.of(regionstr); + return S3Client.builder() + .region(region) + .credentialsProvider(StaticCredentialsProvider.create(awsCreds3)) + .endpointOverride(URI.create(endPointOCE)) + .build(); + } + + public static IamClient getIamClient(Map configs) { + String accessKey = configs.get(RangerS3Constants.ACCESS_KEY); + String secretKey = configs.get(RangerS3Constants.SECRET_KEY); + AwsBasicCredentials awsCreds3 = AwsBasicCredentials.create(accessKey, secretKey); + return IamClient.builder() + .region(Region.AWS_GLOBAL) + .credentialsProvider(StaticCredentialsProvider.create(awsCreds3)) + .build(); + } +} diff --git a/s3-agent/src/main/java/org/apache/ranger/services/s3/client/S3ResourceMgr.java b/s3-agent/src/main/java/org/apache/ranger/services/s3/client/S3ResourceMgr.java new file mode 100644 index 00000000000..016fe7c51cf --- /dev/null +++ b/s3-agent/src/main/java/org/apache/ranger/services/s3/client/S3ResourceMgr.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.services.s3.client; + +import org.apache.ranger.plugin.client.HadoopException; +import org.apache.ranger.plugin.service.ResourceLookupContext; +import org.apache.ranger.plugin.util.TimedEventUtil; +import org.apache.ranger.services.s3.RangerS3Constants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.S3Exception; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +public class S3ResourceMgr { + private static final Logger LOG = LoggerFactory.getLogger(S3ResourceMgr.class); + public static final String PATH = "path"; + + private S3ResourceMgr() { + } + + public static Map connectionTest(String serviceName, Map configs) throws Exception { + Map ret; + LOG.debug("==> S3ResourceMgr.connectionTest ServiceName: " + serviceName); + + try { + ret = S3ClientConnectionMgr.connectionTest(serviceName, configs); + } catch (HadoopException e) { + LOG.error("<== S3ResourceMgr.testConnection Error: " + e.getMessage(), e); + throw e; + } + + LOG.debug("<== S3ResourceMgr.connectionTest Result : " + ret); + return ret; + } + + public static List getS3Resources(String serviceName, Map configs, ResourceLookupContext context, long timeLookup) throws Exception { + List resultList = new ArrayList<>(); + String userInput = context.getUserInput(); + String resource = context.getResourceName(); + Map> resourceMap = context.getResources(); + final List pathList = new ArrayList<>(); + + LOG.debug("==> S3ResourceMgr.getS3Resources ServiceName:{}", serviceName); + + if (resourceMap != null && resource != null && resourceMap.get(PATH) != null && !resourceMap.get(PATH).isEmpty()) { + pathList.addAll(resourceMap.get(PATH)); + } + + if (serviceName != null && userInput != null && !pathList.contains("*") && !userInput.equals("*")) { + try { + if (LOG.isDebugEnabled()) { + LOG.debug("<== S3ResourceMgr.getS3Resources() UserInput: \"" + userInput + "\" resource : " + resource + + " resourceMap: " + resourceMap); + } + S3Client s3 = S3ClientConnectionMgr.getS3client(configs); + + if (s3 != null) { + final Callable> callableObj = new Callable>() { + @Override + public List call() throws Exception { + List resultListInner = new ArrayList<>(); + if (userInput.contains("/")) { + String bucketName = userInput.substring(0, userInput.indexOf("/")); + String path = userInput.substring(userInput.indexOf("/") + 1); + ListObjectsV2Request.Builder listObjectsRequestBuilder = ListObjectsV2Request.builder() + .bucket(bucketName) + .maxKeys(RangerS3Constants.S3_LIST_OBJECTS_MAX_KEYS); + if (!path.isEmpty()) { + listObjectsRequestBuilder.prefix(path.replace("*", "")); // Replace '*' to use only the prefix part + } + ListObjectsV2Request listObjectsRequest = listObjectsRequestBuilder.build(); + ListObjectsV2Response listObjectsResponse; + do { + listObjectsResponse = s3.listObjectsV2(listObjectsRequest); + for (var s3Object : listObjectsResponse.contents()) { + // Check if we've reached the limit before adding more results + if (resultListInner.size() >= RangerS3Constants.MAX_AUTOCOMPLETE_RESULTS) { + if (LOG.isDebugEnabled()) { + LOG.debug("Reached maximum autocomplete results limit of {}, stopping pagination", + RangerS3Constants.MAX_AUTOCOMPLETE_RESULTS); + } + break; + } + String key = s3Object.key(); + String prefixPath = path.replace("*", ""); + if (key.startsWith(prefixPath) && !pathList.contains(prefixPath)) { + resultListInner.add(bucketName + "/" + key); + } + } + + // Short-circuit if we've reached the maximum number of results + if (resultListInner.size() >= RangerS3Constants.MAX_AUTOCOMPLETE_RESULTS) { + break; + } + + // If the response is truncated, set the next continuation token for the next request + String nextContinuationToken = listObjectsResponse.nextContinuationToken(); + if (nextContinuationToken != null) { + listObjectsRequest = listObjectsRequest.toBuilder().continuationToken(nextContinuationToken).build(); + } + } while (listObjectsResponse.isTruncated()); + } + return resultListInner; + } + }; + if (callableObj != null) { + synchronized (s3) { + resultList = TimedEventUtil.timedTask(callableObj, timeLookup, + TimeUnit.MILLISECONDS); + } + } + } + } catch (S3Exception e) { + LOG.error("Failed to list objects for bucket: {}", e.getMessage()); + } catch (Exception e) { + LOG.error("Unable to get S3 resources.", e); + throw e; + } + } + + LOG.info("<== S3ResourceMgr.getS3Resources()"); + return resultList; + } +} diff --git a/security-admin/pom.xml b/security-admin/pom.xml index 4df6c6a8228..6beb8a74abf 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -534,6 +534,11 @@ ranger-util ${project.version} + + org.apache.ranger + s3-agent + ${project.version} + org.apache.ranger ugsync-util @@ -812,6 +817,26 @@ spring-security-web ${springframework.security.version} + + software.amazon.awssdk + auth + ${aws-java-sdk2.version} + + + software.amazon.awssdk + iam + ${aws-java-sdk2.version} + + + software.amazon.awssdk + regions + ${aws-java-sdk2.version} + + + software.amazon.awssdk + s3 + ${aws-java-sdk2.version} + javax.annotation jsr250-api diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 0d3f5bc1bc1..52d9417ddc5 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -19,6 +19,9 @@ package org.apache.ranger.biz; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.IOUtils; @@ -147,6 +150,8 @@ import org.apache.ranger.plugin.util.ServiceTags; import org.apache.ranger.rest.ServiceREST; import org.apache.ranger.rest.TagREST; +import org.apache.ranger.s3.PolicyStatement; +import org.apache.ranger.s3.S3BucketPolicy; import org.apache.ranger.service.RangerAuditFields; import org.apache.ranger.service.RangerDataHistService; import org.apache.ranger.service.RangerPolicyLabelsService; @@ -159,6 +164,8 @@ import org.apache.ranger.service.RangerServiceWithAssignedIdService; import org.apache.ranger.service.XGroupService; import org.apache.ranger.service.XUserService; +import org.apache.ranger.services.s3.RangerS3Constants; +import org.apache.ranger.services.s3.client.S3ClientConnectionMgr; import org.apache.ranger.util.RestUtil; import org.apache.ranger.view.RangerExportPolicyList; import org.apache.ranger.view.RangerExportRoleList; @@ -185,6 +192,16 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.support.TransactionTemplate; +import software.amazon.awssdk.core.exception.SdkServiceException; +import software.amazon.awssdk.services.iam.IamClient; +import software.amazon.awssdk.services.iam.model.GetRoleRequest; +import software.amazon.awssdk.services.iam.model.GetUserRequest; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteBucketPolicyRequest; +import software.amazon.awssdk.services.s3.model.GetBucketPolicyRequest; +import software.amazon.awssdk.services.s3.model.GetBucketPolicyResponse; +import software.amazon.awssdk.services.s3.model.PutBucketPolicyRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; import javax.annotation.PostConstruct; import javax.servlet.ServletOutputStream; @@ -214,6 +231,7 @@ import java.util.Set; import java.util.StringTokenizer; import java.util.TreeSet; +import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -274,76 +292,76 @@ public class ServiceDBStore extends AbstractServiceStore { private static volatile boolean legacyServiceDefsInitDone; private final String optionUgsyncConfigChange = "ugsyncConfigChange"; @Autowired - RangerServiceDefService serviceDefService; + RangerServiceDefService serviceDefService; @Autowired - RangerDaoManager daoMgr; + RangerDaoManager daoMgr; @Autowired - RESTErrorUtil restErrorUtil; + RESTErrorUtil restErrorUtil; @Autowired - RangerServiceService svcService; + RangerServiceService svcService; @Autowired - StringUtil stringUtil; + StringUtil stringUtil; @Autowired - RangerAuditFields rangerAuditFields; + RangerAuditFields rangerAuditFields; @Autowired - RangerPolicyService policyService; + RangerPolicyService policyService; @Autowired RangerPolicyLabelsService policyLabelsService; @Autowired - XUserService xUserService; + XUserService xUserService; @Autowired - XUserMgr xUserMgr; + XUserMgr xUserMgr; @Autowired - XGroupService xGroupService; + XGroupService xGroupService; @Autowired - PolicyRefUpdater policyRefUpdater; + PolicyRefUpdater policyRefUpdater; @Autowired - RangerDataHistService dataHistService; + RangerDataHistService dataHistService; @Autowired @Qualifier(value = "transactionManager") - PlatformTransactionManager txManager; + PlatformTransactionManager txManager; @Autowired - RangerBizUtil bizUtil; + RangerBizUtil bizUtil; @Autowired - RangerPolicyWithAssignedIdService assignedIdPolicyService; + RangerPolicyWithAssignedIdService assignedIdPolicyService; @Autowired - RangerServiceWithAssignedIdService svcServiceWithAssignedId; + RangerServiceWithAssignedIdService svcServiceWithAssignedId; @Autowired - RangerServiceDefWithAssignedIdService svcDefServiceWithAssignedId; + RangerServiceDefWithAssignedIdService svcDefServiceWithAssignedId; @Autowired - RangerFactory factory; + RangerFactory factory; @Autowired - JSONUtil jsonUtil; + JSONUtil jsonUtil; @Autowired - ServiceMgr serviceMgr; + ServiceMgr serviceMgr; @Autowired - AssetMgr assetMgr; + AssetMgr assetMgr; @Autowired - RangerTransactionSynchronizationAdapter transactionSynchronizationAdapter; + RangerTransactionSynchronizationAdapter transactionSynchronizationAdapter; @Autowired - RangerSecurityZoneServiceService securityZoneService; + RangerSecurityZoneServiceService securityZoneService; @Autowired - TagDBStore tagStore; + TagDBStore tagStore; @Autowired - UserMgr userMgr; + UserMgr userMgr; @Autowired - SecurityZoneDBStore securityZoneStore; + SecurityZoneDBStore securityZoneStore; @Autowired - GUIDUtil guidUtil; - private boolean populateExistingBaseFields; + GUIDUtil guidUtil; + private boolean populateExistingBaseFields; private ServicePredicateUtil predicateUtil; - private RangerAdminConfig config; + private RangerAdminConfig config; public static void persistVersionChange(ServiceVersionUpdater serviceVersionUpdater) { - RangerDaoManager daoMgr = serviceVersionUpdater.daoManager; - Long id = serviceVersionUpdater.serviceId; - VERSION_TYPE versionType = serviceVersionUpdater.versionType; - Long nextVersion = 1L; - Date now = new Date(); + RangerDaoManager daoMgr = serviceVersionUpdater.daoManager; + Long id = serviceVersionUpdater.serviceId; + VERSION_TYPE versionType = serviceVersionUpdater.versionType; + Long nextVersion = 1L; + Date now = new Date(); - XXServiceVersionInfoDao serviceVersionInfoDao = daoMgr.getXXServiceVersionInfo(); - XXServiceVersionInfo serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(id); - XXService service = daoMgr.getXXService().getById(id); + XXServiceVersionInfoDao serviceVersionInfoDao = daoMgr.getXXServiceVersionInfo(); + XXServiceVersionInfo serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(id); + XXService service = daoMgr.getXXService().getById(id); if (serviceVersionInfoDbObj != null) { if (versionType == VERSION_TYPE.POLICY_VERSION) { @@ -357,7 +375,8 @@ public static void persistVersionChange(ServiceVersionUpdater serviceVersionUpda serviceVersionInfoDbObj.setTagVersion(nextVersion); serviceVersionInfoDbObj.setTagUpdateTime(now); } else if (versionType == VERSION_TYPE.ROLE_VERSION) { - // get the LatestRoleVersion from the GlobalTable and update ServiceInfo for a service + // get the LatestRoleVersion from the GlobalTable and update ServiceInfo for a + // service XXGlobalStateDao xxGlobalStateDao = daoMgr.getXXGlobalState(); if (xxGlobalStateDao != null) { @@ -366,7 +385,8 @@ public static void persistVersionChange(ServiceVersionUpdater serviceVersionUpda if (roleVersion != null) { nextVersion = roleVersion; } else { - LOG.error("No Global state for 'RoleVersion'. Cannot execute this object:[{}]", serviceVersionUpdater); + LOG.error("No Global state for 'RoleVersion'. Cannot execute this object:[{}]", + serviceVersionUpdater); } serviceVersionInfoDbObj.setRoleVersion(nextVersion); @@ -386,7 +406,8 @@ public static void persistVersionChange(ServiceVersionUpdater serviceVersionUpda serviceVersionInfoDbObj.setGdsVersion(nextVersion); serviceVersionInfoDbObj.setGdsUpdateTime(now); } else { - LOG.error("Unknown VERSION_TYPE:{}. Cannot execute this object:[{}]", versionType, serviceVersionUpdater); + LOG.error("Unknown VERSION_TYPE:{}. Cannot execute this object:[{}]", versionType, + serviceVersionUpdater); return; } @@ -416,7 +437,8 @@ public static void persistVersionChange(ServiceVersionUpdater serviceVersionUpda if (service != null) { if (versionType == VERSION_TYPE.POLICY_VERSION) { - persistChangeLog(service, versionType, serviceVersionInfoDbObj.getPolicyVersion(), serviceVersionUpdater); + persistChangeLog(service, versionType, serviceVersionInfoDbObj.getPolicyVersion(), + serviceVersionUpdater); } else if (versionType == VERSION_TYPE.TAG_VERSION) { persistChangeLog(service, versionType, serviceVersionInfoDbObj.getTagVersion(), serviceVersionUpdater); } @@ -445,33 +467,47 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc XXServiceDef xServiceDef = daoMgr.getXXServiceDef().findByName(serviceDef.getName()); if (xServiceDef != null) { - throw restErrorUtil.createRESTException("service-def with name: " + serviceDef.getName() + " already exists", MessageEnums.ERROR_DUPLICATE_OBJECT); + throw restErrorUtil.createRESTException( + "service-def with name: " + serviceDef.getName() + " already exists", + MessageEnums.ERROR_DUPLICATE_OBJECT); } - List configs = serviceDef.getConfigs(); - List resources = serviceDef.getResources(); + List configs = serviceDef.getConfigs(); + List resources = serviceDef.getResources(); if (CollectionUtils.isNotEmpty(resources)) { - RangerServiceDefValidator validator = new RangerServiceDefValidator(this); - List failures = new ArrayList<>(); - boolean isValidResources = validator.isValidResources(serviceDef, failures, RangerValidator.Action.CREATE); + RangerServiceDefValidator validator = new RangerServiceDefValidator(this); + List failures = new ArrayList<>(); + boolean isValidResources = validator.isValidResources(serviceDef, failures, RangerValidator.Action.CREATE); if (!isValidResources) { - throw restErrorUtil.createRESTException("service-def with name: " + serviceDef.getName() + " has invalid resources:[" + failures + "]", MessageEnums.INVALID_INPUT_DATA); - } - } - - List accessTypes = serviceDef.getAccessTypes(); - List policyConditions = serviceDef.getPolicyConditions(); - List contextEnrichers = serviceDef.getContextEnrichers(); - List enums = serviceDef.getEnums(); - RangerDataMaskDef dataMaskDef = serviceDef.getDataMaskDef(); - RangerRowFilterDef rowFilterDef = serviceDef.getRowFilterDef(); - List dataMaskTypes = dataMaskDef == null || dataMaskDef.getMaskTypes() == null ? new ArrayList<>() : dataMaskDef.getMaskTypes(); - List dataMaskAccessTypes = dataMaskDef == null || dataMaskDef.getAccessTypes() == null ? new ArrayList<>() : dataMaskDef.getAccessTypes(); - List dataMaskResources = dataMaskDef == null || dataMaskDef.getResources() == null ? new ArrayList<>() : dataMaskDef.getResources(); - List rowFilterAccessTypes = rowFilterDef == null || rowFilterDef.getAccessTypes() == null ? new ArrayList<>() : rowFilterDef.getAccessTypes(); - List rowFilterResources = rowFilterDef == null || rowFilterDef.getResources() == null ? new ArrayList<>() : rowFilterDef.getResources(); + throw restErrorUtil.createRESTException( + "service-def with name: " + serviceDef.getName() + " has invalid resources:[" + failures + "]", + MessageEnums.INVALID_INPUT_DATA); + } + } + + List accessTypes = serviceDef.getAccessTypes(); + List policyConditions = serviceDef.getPolicyConditions(); + List contextEnrichers = serviceDef.getContextEnrichers(); + List enums = serviceDef.getEnums(); + RangerDataMaskDef dataMaskDef = serviceDef.getDataMaskDef(); + RangerRowFilterDef rowFilterDef = serviceDef.getRowFilterDef(); + List dataMaskTypes = dataMaskDef == null || dataMaskDef.getMaskTypes() == null + ? new ArrayList<>() + : dataMaskDef.getMaskTypes(); + List dataMaskAccessTypes = dataMaskDef == null || dataMaskDef.getAccessTypes() == null + ? new ArrayList<>() + : dataMaskDef.getAccessTypes(); + List dataMaskResources = dataMaskDef == null || dataMaskDef.getResources() == null + ? new ArrayList<>() + : dataMaskDef.getResources(); + List rowFilterAccessTypes = rowFilterDef == null || rowFilterDef.getAccessTypes() == null + ? new ArrayList<>() + : rowFilterDef.getAccessTypes(); + List rowFilterResources = rowFilterDef == null || rowFilterDef.getResources() == null + ? new ArrayList<>() + : rowFilterDef.getResources(); RangerServiceDefHelper defHelper = new RangerServiceDefHelper(serviceDef, false); @@ -500,15 +536,16 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc serviceDef = serviceDefService.create(serviceDef); } - Long serviceDefId = serviceDef.getId(); - XXServiceDef createdSvcDef = daoMgr.getXXServiceDef().getById(serviceDefId); + Long serviceDefId = serviceDef.getId(); + XXServiceDef createdSvcDef = daoMgr.getXXServiceDef().getById(serviceDefId); XXServiceConfigDefDao xxServiceConfigDao = daoMgr.getXXServiceConfigDef(); for (int i = 0; i < configs.size(); i++) { - RangerServiceConfigDef config = configs.get(i); - XXServiceConfigDef xConfig = new XXServiceConfigDef(); + RangerServiceConfigDef config = configs.get(i); + XXServiceConfigDef xConfig = new XXServiceConfigDef(); - xConfig = serviceDefService.populateRangerServiceConfigDefToXX(config, xConfig, createdSvcDef, RangerServiceDefService.OPERATION_CREATE_CONTEXT); + xConfig = serviceDefService.populateRangerServiceConfigDefToXX(config, xConfig, createdSvcDef, + RangerServiceDefService.OPERATION_CREATE_CONTEXT); xConfig.setOrder(i); @@ -518,12 +555,13 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc XXResourceDefDao xxResDefDao = daoMgr.getXXResourceDef(); for (int i = 0; i < resources.size(); i++) { - RangerResourceDef resource = resources.get(i); - XXResourceDef parent = xxResDefDao.findByNameAndServiceDefId(resource.getParent(), serviceDefId); - Long parentId = (parent != null) ? parent.getId() : null; - XXResourceDef xResource = new XXResourceDef(); + RangerResourceDef resource = resources.get(i); + XXResourceDef parent = xxResDefDao.findByNameAndServiceDefId(resource.getParent(), serviceDefId); + Long parentId = (parent != null) ? parent.getId() : null; + XXResourceDef xResource = new XXResourceDef(); - xResource = serviceDefService.populateRangerResourceDefToXX(resource, xResource, createdSvcDef, RangerServiceDefService.OPERATION_CREATE_CONTEXT); + xResource = serviceDefService.populateRangerResourceDefToXX(resource, xResource, createdSvcDef, + RangerServiceDefService.OPERATION_CREATE_CONTEXT); xResource.setOrder(i); xResource.setParent(parentId); @@ -534,16 +572,17 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc XXAccessTypeDefDao xxATDDao = daoMgr.getXXAccessTypeDef(); for (int i = 0; i < accessTypes.size(); i++) { - RangerAccessTypeDef accessType = accessTypes.get(i); - XXAccessTypeDef xAccessType = new XXAccessTypeDef(); + RangerAccessTypeDef accessType = accessTypes.get(i); + XXAccessTypeDef xAccessType = new XXAccessTypeDef(); - xAccessType = serviceDefService.populateRangerAccessTypeDefToXX(accessType, xAccessType, createdSvcDef, RangerServiceDefService.OPERATION_CREATE_CONTEXT); + xAccessType = serviceDefService.populateRangerAccessTypeDefToXX(accessType, xAccessType, createdSvcDef, + RangerServiceDefService.OPERATION_CREATE_CONTEXT); xAccessType.setOrder(i); xAccessType = xxATDDao.create(xAccessType); - Collection impliedGrants = accessType.getImpliedGrants(); + Collection impliedGrants = accessType.getImpliedGrants(); XXAccessTypeDefGrantsDao xxATDGrantDao = daoMgr.getXXAccessTypeDefGrants(); for (String impliedGrant : impliedGrants) { @@ -559,10 +598,11 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc XXPolicyConditionDefDao xxPolCondDao = daoMgr.getXXPolicyConditionDef(); for (int i = 0; i < policyConditions.size(); i++) { - RangerPolicyConditionDef policyCondition = policyConditions.get(i); - XXPolicyConditionDef xPolicyCondition = new XXPolicyConditionDef(); + RangerPolicyConditionDef policyCondition = policyConditions.get(i); + XXPolicyConditionDef xPolicyCondition = new XXPolicyConditionDef(); - xPolicyCondition = serviceDefService.populateRangerPolicyConditionDefToXX(policyCondition, xPolicyCondition, createdSvcDef, RangerServiceDefService.OPERATION_CREATE_CONTEXT); + xPolicyCondition = serviceDefService.populateRangerPolicyConditionDefToXX(policyCondition, xPolicyCondition, + createdSvcDef, RangerServiceDefService.OPERATION_CREATE_CONTEXT); xPolicyCondition.setOrder(i); @@ -572,10 +612,11 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc XXContextEnricherDefDao xxContextEnricherDao = daoMgr.getXXContextEnricherDef(); for (int i = 0; i < contextEnrichers.size(); i++) { - RangerContextEnricherDef contextEnricher = contextEnrichers.get(i); - XXContextEnricherDef xContextEnricher = new XXContextEnricherDef(); + RangerContextEnricherDef contextEnricher = contextEnrichers.get(i); + XXContextEnricherDef xContextEnricher = new XXContextEnricherDef(); - xContextEnricher = serviceDefService.populateRangerContextEnricherDefToXX(contextEnricher, xContextEnricher, createdSvcDef, RangerServiceDefService.OPERATION_CREATE_CONTEXT); + xContextEnricher = serviceDefService.populateRangerContextEnricherDefToXX(contextEnricher, xContextEnricher, + createdSvcDef, RangerServiceDefService.OPERATION_CREATE_CONTEXT); xContextEnricher.setOrder(i); @@ -587,17 +628,19 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc for (RangerEnumDef vEnum : enums) { XXEnumDef xEnum = new XXEnumDef(); - xEnum = serviceDefService.populateRangerEnumDefToXX(vEnum, xEnum, createdSvcDef, RangerServiceDefService.OPERATION_CREATE_CONTEXT); + xEnum = serviceDefService.populateRangerEnumDefToXX(vEnum, xEnum, createdSvcDef, + RangerServiceDefService.OPERATION_CREATE_CONTEXT); xEnum = xxEnumDefDao.create(xEnum); - List elements = vEnum.getElements(); - XXEnumElementDefDao xxEnumEleDefDao = daoMgr.getXXEnumElementDef(); + List elements = vEnum.getElements(); + XXEnumElementDefDao xxEnumEleDefDao = daoMgr.getXXEnumElementDef(); for (int i = 0; i < elements.size(); i++) { - RangerEnumElementDef element = elements.get(i); - XXEnumElementDef xElement = new XXEnumElementDef(); + RangerEnumElementDef element = elements.get(i); + XXEnumElementDef xElement = new XXEnumElementDef(); - xElement = serviceDefService.populateRangerEnumElementDefToXX(element, xElement, xEnum, RangerServiceDefService.OPERATION_CREATE_CONTEXT); + xElement = serviceDefService.populateRangerEnumElementDefToXX(element, xElement, xEnum, + RangerServiceDefService.OPERATION_CREATE_CONTEXT); xElement.setOrder(i); @@ -608,10 +651,11 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc XXDataMaskTypeDefDao xxDataMaskDefDao = daoMgr.getXXDataMaskTypeDef(); for (int i = 0; i < dataMaskTypes.size(); i++) { - RangerDataMaskTypeDef dataMask = dataMaskTypes.get(i); - XXDataMaskTypeDef xDataMaskDef = new XXDataMaskTypeDef(); + RangerDataMaskTypeDef dataMask = dataMaskTypes.get(i); + XXDataMaskTypeDef xDataMaskDef = new XXDataMaskTypeDef(); - xDataMaskDef = serviceDefService.populateRangerDataMaskDefToXX(dataMask, xDataMaskDef, createdSvcDef, RangerServiceDefService.OPERATION_CREATE_CONTEXT); + xDataMaskDef = serviceDefService.populateRangerDataMaskDefToXX(dataMask, xDataMaskDef, createdSvcDef, + RangerServiceDefService.OPERATION_CREATE_CONTEXT); xDataMaskDef.setOrder(i); @@ -622,18 +666,22 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc for (RangerAccessTypeDef accessType : dataMaskAccessTypes) { if (!isAccessTypeInList(accessType.getName(), xxAccessTypeDefs)) { - throw restErrorUtil.createRESTException("accessType with name: " + accessType.getName() + " does not exists", MessageEnums.DATA_NOT_FOUND); + throw restErrorUtil.createRESTException( + "accessType with name: " + accessType.getName() + " does not exists", + MessageEnums.DATA_NOT_FOUND); } } for (RangerAccessTypeDef accessType : rowFilterAccessTypes) { if (!isAccessTypeInList(accessType.getName(), xxAccessTypeDefs)) { - throw restErrorUtil.createRESTException("accessType with name: " + accessType.getName() + " does not exists", MessageEnums.DATA_NOT_FOUND); + throw restErrorUtil.createRESTException( + "accessType with name: " + accessType.getName() + " does not exists", + MessageEnums.DATA_NOT_FOUND); } } for (XXAccessTypeDef xxAccessTypeDef : xxAccessTypeDefs) { - String dataMaskOptions = null; + String dataMaskOptions = null; String rowFilterOptions = null; for (RangerAccessTypeDef accessTypeDef : dataMaskAccessTypes) { @@ -650,7 +698,8 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc } } - if (!StringUtils.equals(dataMaskOptions, xxAccessTypeDef.getDataMaskOptions()) || !StringUtils.equals(rowFilterOptions, xxAccessTypeDef.getRowFilterOptions())) { + if (!StringUtils.equals(dataMaskOptions, xxAccessTypeDef.getDataMaskOptions()) + || !StringUtils.equals(rowFilterOptions, xxAccessTypeDef.getRowFilterOptions())) { xxAccessTypeDef.setDataMaskOptions(dataMaskOptions); xxAccessTypeDef.setRowFilterOptions(rowFilterOptions); @@ -662,18 +711,20 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc for (RangerResourceDef resource : dataMaskResources) { if (!isResourceInList(resource.getName(), xxResourceDefs)) { - throw restErrorUtil.createRESTException("resource with name: " + resource.getName() + " does not exists", MessageEnums.DATA_NOT_FOUND); + throw restErrorUtil.createRESTException( + "resource with name: " + resource.getName() + " does not exists", MessageEnums.DATA_NOT_FOUND); } } for (RangerResourceDef resource : rowFilterResources) { if (!isResourceInList(resource.getName(), xxResourceDefs)) { - throw restErrorUtil.createRESTException("resource with name: " + resource.getName() + " does not exists", MessageEnums.DATA_NOT_FOUND); + throw restErrorUtil.createRESTException( + "resource with name: " + resource.getName() + " does not exists", MessageEnums.DATA_NOT_FOUND); } } for (XXResourceDef xxResourceDef : xxResourceDefs) { - String dataMaskOptions = null; + String dataMaskOptions = null; String rowFilterOptions = null; for (RangerResourceDef resource : dataMaskResources) { @@ -690,7 +741,8 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc } } - if (!StringUtils.equals(dataMaskOptions, xxResourceDef.getDataMaskOptions()) || !StringUtils.equals(rowFilterOptions, xxResourceDef.getRowFilterOptions())) { + if (!StringUtils.equals(dataMaskOptions, xxResourceDef.getDataMaskOptions()) + || !StringUtils.equals(rowFilterOptions, xxResourceDef.getRowFilterOptions())) { xxResourceDef.setDataMaskOptions(dataMaskOptions); xxResourceDef.setRowFilterOptions(rowFilterOptions); @@ -713,33 +765,42 @@ public RangerServiceDef createServiceDef(RangerServiceDef serviceDef) throws Exc public RangerServiceDef updateServiceDef(RangerServiceDef serviceDef) throws Exception { LOG.debug("==> ServiceDBStore.updateServiceDef({})", serviceDef); - Long serviceDefId = serviceDef.getId(); - XXServiceDef existing = daoMgr.getXXServiceDef().getById(serviceDefId); + Long serviceDefId = serviceDef.getId(); + XXServiceDef existing = daoMgr.getXXServiceDef().getById(serviceDefId); if (existing == null) { - throw restErrorUtil.createRESTException("no service-def exists with ID=" + serviceDef.getId(), MessageEnums.DATA_NOT_FOUND); + throw restErrorUtil.createRESTException("no service-def exists with ID=" + serviceDef.getId(), + MessageEnums.DATA_NOT_FOUND); } - String existingName = existing.getName(); - boolean renamed = !StringUtils.equalsIgnoreCase(serviceDef.getName(), existingName); + String existingName = existing.getName(); + boolean renamed = !StringUtils.equalsIgnoreCase(serviceDef.getName(), existingName); if (renamed) { XXServiceDef renamedSVCDef = daoMgr.getXXServiceDef().findByName(serviceDef.getName()); if (renamedSVCDef != null) { - throw restErrorUtil.createRESTException("another service-def already exists with name '" + serviceDef.getName() + "'. ID=" + renamedSVCDef.getId(), MessageEnums.DATA_NOT_UPDATABLE); - } - } - - List configs = serviceDef.getConfigs() != null ? serviceDef.getConfigs() : new ArrayList<>(); - List resources = serviceDef.getResources() != null ? serviceDef.getResources() : new ArrayList<>(); - List accessTypes = serviceDef.getAccessTypes() != null ? serviceDef.getAccessTypes() : new ArrayList<>(); - List policyConditions = serviceDef.getPolicyConditions() != null ? serviceDef.getPolicyConditions() : new ArrayList<>(); - List contextEnrichers = serviceDef.getContextEnrichers() != null ? serviceDef.getContextEnrichers() : new ArrayList<>(); - List enums = serviceDef.getEnums() != null ? serviceDef.getEnums() : new ArrayList<>(); - RangerDataMaskDef dataMaskDef = serviceDef.getDataMaskDef(); - RangerRowFilterDef rowFilterDef = serviceDef.getRowFilterDef(); - RangerServiceDefHelper defHelper = new RangerServiceDefHelper(serviceDef, false); + throw restErrorUtil.createRESTException("another service-def already exists with name '" + + serviceDef.getName() + "'. ID=" + renamedSVCDef.getId(), MessageEnums.DATA_NOT_UPDATABLE); + } + } + + List configs = serviceDef.getConfigs() != null ? serviceDef.getConfigs() + : new ArrayList<>(); + List resources = serviceDef.getResources() != null ? serviceDef.getResources() + : new ArrayList<>(); + List accessTypes = serviceDef.getAccessTypes() != null ? serviceDef.getAccessTypes() + : new ArrayList<>(); + List policyConditions = serviceDef.getPolicyConditions() != null + ? serviceDef.getPolicyConditions() + : new ArrayList<>(); + List contextEnrichers = serviceDef.getContextEnrichers() != null + ? serviceDef.getContextEnrichers() + : new ArrayList<>(); + List enums = serviceDef.getEnums() != null ? serviceDef.getEnums() : new ArrayList<>(); + RangerDataMaskDef dataMaskDef = serviceDef.getDataMaskDef(); + RangerRowFilterDef rowFilterDef = serviceDef.getRowFilterDef(); + RangerServiceDefHelper defHelper = new RangerServiceDefHelper(serviceDef, false); defHelper.patchServiceDefWithDefaultValues(); @@ -751,7 +812,8 @@ public RangerServiceDef updateServiceDef(RangerServiceDef serviceDef) throws Exc XXServiceDef createdSvcDef = daoMgr.getXXServiceDef().getById(serviceDefId); - updateChildObjectsOfServiceDef(createdSvcDef, configs, resources, accessTypes, policyConditions, contextEnrichers, enums, dataMaskDef, rowFilterDef); + updateChildObjectsOfServiceDef(createdSvcDef, configs, resources, accessTypes, policyConditions, + contextEnrichers, enums, dataMaskDef, rowFilterDef); RangerServiceDef updatedSvcDef = getServiceDef(serviceDefId); @@ -772,24 +834,31 @@ public void deleteServiceDef(Long serviceDefId, Boolean forceDelete) throws Exce UserSessionBase session = ContextUtil.getCurrentUserSession(); if (session == null) { - throw restErrorUtil.createRESTException("UserSession cannot be null, only Admin can update service-def", MessageEnums.OPER_NO_PERMISSION); + throw restErrorUtil.createRESTException("UserSession cannot be null, only Admin can update service-def", + MessageEnums.OPER_NO_PERMISSION); } if (!session.isKeyAdmin() && !session.isUserAdmin()) { - throw restErrorUtil.createRESTException("User is not allowed to update service-def, only Admin can update service-def", MessageEnums.OPER_NO_PERMISSION); + throw restErrorUtil.createRESTException( + "User is not allowed to update service-def, only Admin can update service-def", + MessageEnums.OPER_NO_PERMISSION); } RangerServiceDef serviceDef = getServiceDef(serviceDefId); if (serviceDef == null) { - throw restErrorUtil.createRESTException("No Service Definiton found for Id: " + serviceDefId, MessageEnums.DATA_NOT_FOUND); + throw restErrorUtil.createRESTException("No Service Definiton found for Id: " + serviceDefId, + MessageEnums.DATA_NOT_FOUND); } List serviceList = daoMgr.getXXService().findByServiceDefId(serviceDefId); if (!forceDelete) { if (CollectionUtils.isNotEmpty(serviceList)) { - throw restErrorUtil.createRESTException("Services exists under given service definition, can't delete Service-Def: " + serviceDef.getName(), MessageEnums.OPER_NOT_ALLOWED_FOR_ENTITY); + throw restErrorUtil.createRESTException( + "Services exists under given service definition, can't delete Service-Def: " + + serviceDef.getName(), + MessageEnums.OPER_NOT_ALLOWED_FOR_ENTITY); } } @@ -799,7 +868,7 @@ public void deleteServiceDef(Long serviceDefId, Boolean forceDelete) throws Exce } } - XXDataMaskTypeDefDao dataMaskDao = daoMgr.getXXDataMaskTypeDef(); + XXDataMaskTypeDefDao dataMaskDao = daoMgr.getXXDataMaskTypeDef(); List dataMaskDefs = dataMaskDao.findByServiceDefId(serviceDefId); for (XXDataMaskTypeDef dataMaskDef : dataMaskDefs) { @@ -812,14 +881,14 @@ public void deleteServiceDef(Long serviceDefId, Boolean forceDelete) throws Exce deleteXXAccessTypeDef(accessType); } - XXContextEnricherDefDao xContextEnricherDao = daoMgr.getXXContextEnricherDef(); - List contextEnrichers = xContextEnricherDao.findByServiceDefId(serviceDefId); + XXContextEnricherDefDao xContextEnricherDao = daoMgr.getXXContextEnricherDef(); + List contextEnrichers = xContextEnricherDao.findByServiceDefId(serviceDefId); for (XXContextEnricherDef context : contextEnrichers) { xContextEnricherDao.remove(context); } - XXEnumDefDao enumDefDao = daoMgr.getXXEnumDef(); + XXEnumDefDao enumDefDao = daoMgr.getXXEnumDef(); List enumDefList = enumDefDao.findByServiceDefId(serviceDefId); for (XXEnumDef enumDef : enumDefList) { @@ -832,11 +901,12 @@ public void deleteServiceDef(Long serviceDefId, Boolean forceDelete) throws Exce enumDefDao.remove(enumDef); } - XXPolicyConditionDefDao policyCondDao = daoMgr.getXXPolicyConditionDef(); + XXPolicyConditionDefDao policyCondDao = daoMgr.getXXPolicyConditionDef(); List policyCondList = policyCondDao.findByServiceDefId(serviceDefId); for (XXPolicyConditionDef policyCond : policyCondList) { - List xxPolicyRefConditions = daoMgr.getXXPolicyRefCondition().findByConditionDefId(policyCond.getId()); + List xxPolicyRefConditions = daoMgr.getXXPolicyRefCondition() + .findByConditionDefId(policyCond.getId()); for (XXPolicyRefCondition xxPolicyRefCondition : xxPolicyRefConditions) { daoMgr.getXXPolicyRefCondition().remove(xxPolicyRefCondition); @@ -851,7 +921,7 @@ public void deleteServiceDef(Long serviceDefId, Boolean forceDelete) throws Exce deleteXXResourceDef(resDef); } - XXServiceConfigDefDao configDefDao = daoMgr.getXXServiceConfigDef(); + XXServiceConfigDefDao configDefDao = daoMgr.getXXServiceConfigDef(); List configDefList = configDefDao.findByServiceDefId(serviceDefId); for (XXServiceConfigDef configDef : configDefList) { @@ -863,7 +933,8 @@ public void deleteServiceDef(Long serviceDefId, Boolean forceDelete) throws Exce if (version == null) { version = 1L; - LOG.info("Found Version Value: `null`, so setting value of version to 1, While updating object, version should not be null."); + LOG.info( + "Found Version Value: `null`, so setting value of version to 1, While updating object, version should not be null."); } else { version = version + 1; } @@ -896,8 +967,8 @@ public RangerServiceDef getServiceDef(Long id) throws Exception { public RangerServiceDef getServiceDefByName(String name) throws Exception { LOG.debug("==> ServiceDBStore.getServiceDefByName({})", name); - RangerServiceDef ret = null; - XXServiceDef xServiceDef = daoMgr.getXXServiceDef().findByName(name); + RangerServiceDef ret = null; + XXServiceDef xServiceDef = daoMgr.getXXServiceDef().findByName(name); if (xServiceDef != null) { ret = serviceDefService.getPopulatedViewObject(xServiceDef); @@ -910,14 +981,15 @@ public RangerServiceDef getServiceDefByName(String name) throws Exception { /** * @param displayName - * @return {@link RangerServiceDef} - service using display name if present in DB, null otherwise. + * @return {@link RangerServiceDef} - service using display name if present in + * DB, null otherwise. */ @Override public RangerServiceDef getServiceDefByDisplayName(String displayName) { LOG.debug("==> ServiceDBStore.getServiceDefByDisplayName({})", displayName); - RangerServiceDef ret = null; - XXServiceDef xServiceDef = daoMgr.getXXServiceDef().findByDisplayName(displayName); + RangerServiceDef ret = null; + XXServiceDef xServiceDef = daoMgr.getXXServiceDef().findByDisplayName(displayName); if (xServiceDef != null) { ret = serviceDefService.getPopulatedViewObject(xServiceDef); @@ -948,12 +1020,13 @@ public RangerService createService(RangerService service) throws Exception { LOG.debug("==> ServiceDBStore.createService({})", service); if (service == null) { - throw restErrorUtil.createRESTException("Service object cannot be null.", MessageEnums.ERROR_CREATING_OBJECT); + throw restErrorUtil.createRESTException("Service object cannot be null.", + MessageEnums.ERROR_CREATING_OBJECT); } - boolean createDefaultPolicy = true; - Map configs = service.getConfigs(); - Map validConfigs = validateRequiredConfigParams(service, configs); + boolean createDefaultPolicy = true; + Map configs = service.getConfigs(); + Map validConfigs = validateRequiredConfigParams(service, configs); if (validConfigs == null) { LOG.debug("==> ConfigParams cannot be null, ServiceDBStore.createService({})", service); @@ -987,12 +1060,12 @@ public RangerService createService(RangerService service) throws Exception { Set passwordConfigKeys = getPasswordConfigKeys(service.getType()); for (Entry configMap : validConfigs.entrySet()) { - String configKey = configMap.getKey(); + String configKey = configMap.getKey(); String configValue = configMap.getValue(); if (StringUtils.equalsIgnoreCase(configKey, "username")) { String userName = stringUtil.getValidUserName(configValue); - XXUser xxUser = daoMgr.getXXUser().findByUserName(userName); + XXUser xxUser = daoMgr.getXXUser().findByUserName(userName); if (xxUser != null) { xUserService.populateViewBean(xxUser); @@ -1000,7 +1073,9 @@ public RangerService createService(RangerService service) throws Exception { UserSessionBase usb = ContextUtil.getCurrentUserSession(); if (usb != null && !usb.isUserAdmin() && !usb.isSpnegoEnabled()) { - throw restErrorUtil.createRESTException("User does not exist with given username: [" + userName + "] please use existing user", MessageEnums.OPER_NO_PERMISSION); + throw restErrorUtil.createRESTException( + "User does not exist with given username: [" + userName + "] please use existing user", + MessageEnums.OPER_NO_PERMISSION); } xUserMgr.createServiceConfigUser(userName); @@ -1041,7 +1116,8 @@ public RangerService createService(RangerService service) throws Exception { RangerService createdService = svcService.getPopulatedViewObject(xCreatedService); if (createdService == null) { - throw restErrorUtil.createRESTException("Could not create service - Internal error ", MessageEnums.ERROR_CREATING_OBJECT); + throw restErrorUtil.createRESTException("Could not create service - Internal error ", + MessageEnums.ERROR_CREATING_OBJECT); } dataHistService.createObjectDataHistory(createdService, RangerDataHistService.ACTION_CREATE); @@ -1062,7 +1138,8 @@ public RangerService updateService(RangerService service, Map op XXService xExisting = daoMgr.getXXService().getById(service.getId()); if (xExisting == null) { - throw restErrorUtil.createRESTException("no service exists with ID=" + service.getId(), MessageEnums.DATA_NOT_FOUND); + throw restErrorUtil.createRESTException("no service exists with ID=" + service.getId(), + MessageEnums.DATA_NOT_FOUND); } RangerService existing = svcService.getPopulatedViewObject(xExisting); @@ -1075,23 +1152,31 @@ public RangerService updateService(RangerService service, Map op XXService newNameService = daoMgr.getXXService().findByName(service.getName()); if (newNameService != null) { - throw restErrorUtil.createRESTException("another service already exists with name '" + service.getName() + "'. ID=" + newNameService.getId(), MessageEnums.DATA_NOT_UPDATABLE); + throw restErrorUtil.createRESTException("another service already exists with name '" + service.getName() + + "'. ID=" + newNameService.getId(), MessageEnums.DATA_NOT_UPDATABLE); } - long countOfTaggedResources = daoMgr.getXXServiceResource().countTaggedResourcesInServiceId(existing.getId()); + long countOfTaggedResources = daoMgr.getXXServiceResource() + .countTaggedResourcesInServiceId(existing.getId()); - Boolean isForceRename = options != null && options.get(ServiceStore.OPTION_FORCE_RENAME) != null ? (Boolean) options.get(ServiceStore.OPTION_FORCE_RENAME) : Boolean.FALSE; + Boolean isForceRename = options != null && options.get(ServiceStore.OPTION_FORCE_RENAME) != null + ? (Boolean) options.get(ServiceStore.OPTION_FORCE_RENAME) + : Boolean.FALSE; if (countOfTaggedResources != 0L) { if (isForceRename) { - LOG.warn("Forcing the renaming of service from {} to {} although it is associated with {} service-resources!", existingName, service.getName(), countOfTaggedResources); + LOG.warn( + "Forcing the renaming of service from {} to {} although it is associated with {} service-resources!", + existingName, service.getName(), countOfTaggedResources); } else { - throw restErrorUtil.createRESTException("Service " + existingName + " cannot be renamed, as it has associated service-resources", MessageEnums.DATA_NOT_UPDATABLE); + throw restErrorUtil.createRESTException( + "Service " + existingName + " cannot be renamed, as it has associated service-resources", + MessageEnums.DATA_NOT_UPDATABLE); } } } - Map configs = service.getConfigs(); + Map configs = service.getConfigs(); Map validConfigs = validateRequiredConfigParams(service, configs); if (validConfigs == null) { @@ -1101,9 +1186,10 @@ public RangerService updateService(RangerService service, Map op } boolean hasTagServiceValueChanged = false; - String existingTagService = existing.getTagService(); - String newTagServiceName = service.getTagService(); // null for old clients; empty string to remove existing association - Long newTagServiceId = null; + String existingTagService = existing.getTagService(); + String newTagServiceName = service.getTagService(); // null for old clients; empty string to remove existing + // association + Long newTagServiceId = null; if (newTagServiceName == null) { // old client; don't update existing tagService if (existingTagService != null) { @@ -1111,7 +1197,9 @@ public RangerService updateService(RangerService service, Map op service.setTagService(newTagServiceName); - LOG.info("ServiceDBStore.updateService(id={}; name={}): tagService is null; using existing tagService '{}'", service.getId(), service.getName(), newTagServiceName); + LOG.info( + "ServiceDBStore.updateService(id={}; name={}): tagService is null; using existing tagService '{}'", + service.getId(), service.getName(), newTagServiceName); } } @@ -1119,9 +1207,11 @@ public RangerService updateService(RangerService service, Map op RangerService tmp = getServiceByName(newTagServiceName); if (tmp == null || !EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME.equals(tmp.getType())) { - LOG.debug("ServiceDBStore.updateService() - {} does not refer to a valid tag service.({})", newTagServiceName, service); + LOG.debug("ServiceDBStore.updateService() - {} does not refer to a valid tag service.({})", + newTagServiceName, service); - throw restErrorUtil.createRESTException("Invalid tag service name " + newTagServiceName, MessageEnums.ERROR_CREATING_OBJECT); + throw restErrorUtil.createRESTException("Invalid tag service name " + newTagServiceName, + MessageEnums.ERROR_CREATING_OBJECT); } else { newTagServiceId = tmp.getId(); } @@ -1137,8 +1227,8 @@ public RangerService updateService(RangerService service, Map op boolean hasIsEnabledChanged = !existing.getIsEnabled().equals(service.getIsEnabled()); - List dbConfigMaps = daoMgr.getXXServiceConfigMap().findByServiceId(service.getId()); - boolean hasServiceConfigForPluginChanged = hasServiceConfigForPluginChanged(dbConfigMaps, validConfigs); + List dbConfigMaps = daoMgr.getXXServiceConfigMap().findByServiceId(service.getId()); + boolean hasServiceConfigForPluginChanged = hasServiceConfigForPluginChanged(dbConfigMaps, validConfigs); svcService.createTransactionLog(service, existing, RangerServiceService.OPERATION_UPDATE_CONTEXT); @@ -1155,8 +1245,11 @@ public RangerService updateService(RangerService service, Map op service = svcService.update(service); - Boolean isUgsyncConfigChange = options != null && options.get(optionUgsyncConfigChange) != null ? (Boolean) options.get(optionUgsyncConfigChange) : Boolean.FALSE; - if (hasTagServiceValueChanged || hasIsEnabledChanged || hasServiceConfigForPluginChanged || isUgsyncConfigChange) { + Boolean isUgsyncConfigChange = options != null && options.get(optionUgsyncConfigChange) != null + ? (Boolean) options.get(optionUgsyncConfigChange) + : Boolean.FALSE; + if (hasTagServiceValueChanged || hasIsEnabledChanged || hasServiceConfigForPluginChanged + || isUgsyncConfigChange) { updatePolicyVersion(service, RangerPolicyDelta.CHANGE_TYPE_SERVICE_CHANGE, null, false); } } @@ -1176,12 +1269,12 @@ public RangerService updateService(RangerService service, Map op XXServiceConfigMapDao xConfMapDao = daoMgr.getXXServiceConfigMap(); for (Entry configMap : validConfigs.entrySet()) { - String configKey = configMap.getKey(); + String configKey = configMap.getKey(); String configValue = configMap.getValue(); if (StringUtils.equalsIgnoreCase(configKey, "username")) { String userName = stringUtil.getValidUserName(configValue); - XXUser xxUser = daoMgr.getXXUser().findByUserName(userName); + XXUser xxUser = daoMgr.getXXUser().findByUserName(userName); if (xxUser != null) { xUserService.populateViewBean(xxUser); @@ -1189,7 +1282,9 @@ public RangerService updateService(RangerService service, Map op UserSessionBase usb = ContextUtil.getCurrentUserSession(); if (usb != null && !usb.isUserAdmin()) { - throw restErrorUtil.createRESTException("User does not exist with given username: [" + userName + "] please use existing user", MessageEnums.OPER_NO_PERMISSION); + throw restErrorUtil.createRESTException( + "User does not exist with given username: [" + userName + "] please use existing user", + MessageEnums.OPER_NO_PERMISSION); } xUserMgr.createServiceConfigUser(userName); @@ -1204,9 +1299,11 @@ public RangerService updateService(RangerService service, Map op PasswordUtils util = PasswordUtils.build(oldPassword); if (!util.getCryptAlgo().equalsIgnoreCase(CRYPT_ALGO)) { - String decryptedPwd = PasswordUtils.decryptPassword(oldPassword); - String paddingString = Joiner.on(",").skipNulls().join(CRYPT_ALGO, new String(util.getEncryptKey()), new String(util.getSalt()), util.getIterationCount(), PasswordUtils.generateIvIfNeeded(CRYPT_ALGO)); - String encryptedPwd = PasswordUtils.encryptPassword(paddingString + "," + decryptedPwd); + String decryptedPwd = PasswordUtils.decryptPassword(oldPassword); + String paddingString = Joiner.on(",").skipNulls().join(CRYPT_ALGO, + new String(util.getEncryptKey()), new String(util.getSalt()), + util.getIterationCount(), PasswordUtils.generateIvIfNeeded(CRYPT_ALGO)); + String encryptedPwd = PasswordUtils.encryptPassword(paddingString + "," + decryptedPwd); String newDecryptedPwd = PasswordUtils.decryptPassword(paddingString + "," + encryptedPwd); if (StringUtils.equals(newDecryptedPwd, decryptedPwd)) { @@ -1219,9 +1316,10 @@ public RangerService updateService(RangerService service, Map op configValue = oldPassword; } } else { - String paddingString = Joiner.on(",").skipNulls().join(CRYPT_ALGO, ENCRYPT_KEY, SALT, ITERATION_COUNT, PasswordUtils.generateIvIfNeeded(CRYPT_ALGO)); - String encryptedPwd = PasswordUtils.encryptPassword(paddingString + "," + configValue); - String decryptedPwd = PasswordUtils.decryptPassword(paddingString + "," + encryptedPwd); + String paddingString = Joiner.on(",").skipNulls().join(CRYPT_ALGO, ENCRYPT_KEY, SALT, + ITERATION_COUNT, PasswordUtils.generateIvIfNeeded(CRYPT_ALGO)); + String encryptedPwd = PasswordUtils.encryptPassword(paddingString + "," + configValue); + String decryptedPwd = PasswordUtils.decryptPassword(paddingString + "," + encryptedPwd); if (StringUtils.equals(decryptedPwd, configValue)) { configValue = paddingString + "," + encryptedPwd; @@ -1260,10 +1358,36 @@ public void deleteService(Long id) throws Exception { } // Manage zone - disassociateZonesForService(service); //RANGER-3016 + disassociateZonesForService(service); // RANGER-3016 List policyIds = daoMgr.getXXPolicy().findPolicyIdsByServiceId(service.getId()); + // Handle S3 bucket policy cleanup BEFORE deleting policies + if (CollectionUtils.isNotEmpty(policyIds) && + service.getType() != null && + service.getType().equalsIgnoreCase(RangerS3Constants.S3)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Cleaning up S3 bucket policies for service: " + service.getName()); + } + + try { + // Get all policies that will be deleted + List policiesToDelete = new ArrayList<>(); + for (Long policyID : policyIds) { + RangerPolicy policy = getPolicy(policyID); + if (policy != null) { + policiesToDelete.add(policy); + } + } + + // Clean up S3 bucket policies for all affected buckets + cleanupS3BucketPoliciesForService(service, policiesToDelete); + } catch (Exception e) { + LOG.error("Error cleaning up S3 bucket policies for service: " + service.getName(), e); + // Continue with service deletion even if S3 cleanup fails + } + } + if (CollectionUtils.isNotEmpty(policyIds)) { long totalDeletedPolicies = 0; @@ -1283,8 +1407,8 @@ public void deleteService(Long id) throws Exception { bizUtil.bulkModeOnlyFlushAndClear(); } - XXServiceConfigMapDao configDao = daoMgr.getXXServiceConfigMap(); - List configs = configDao.findByServiceId(service.getId()); + XXServiceConfigMapDao configDao = daoMgr.getXXServiceConfigMap(); + List configs = configDao.findByServiceId(service.getId()); for (XXServiceConfigMap configMap : configs) { configDao.remove(configMap); @@ -1298,7 +1422,8 @@ public void deleteService(Long id) throws Exception { if (version == null) { version = 1L; - LOG.info("Found Version Value: `null`, so setting value of version to 1, While updating object, version should not be null."); + LOG.info( + "Found Version Value: `null`, so setting value of version to 1, While updating object, version should not be null."); } else { version = version + 1; } @@ -1311,7 +1436,8 @@ public void deleteService(Long id) throws Exception { svcService.createTransactionLog(service, null, RangerServiceService.OPERATION_DELETE_CONTEXT); - //During the servie deletion ,we need to clear the RangerServicePoliciesCache,RangerServiceTagsCache for the given serviceName. + // During the servie deletion ,we need to clear the + // RangerServicePoliciesCache,RangerServiceTagsCache for the given serviceName. resetPolicyCache(service.getName()); tagStore.resetTagCache(service.getName()); @@ -1321,7 +1447,7 @@ public void deleteService(Long id) throws Exception { public boolean serviceExists(String name) { LOG.debug("==> ServiceDBStore.serviceExists({})", name); - Long id = daoMgr.getXXService().findIdByName(name); + Long id = daoMgr.getXXService().findIdByName(name); boolean ret = id != null; LOG.debug("<== ServiceDBStore.serviceExists({}): ret={}", name, ret); @@ -1336,7 +1462,8 @@ public RangerService getService(Long id) throws Exception { UserSessionBase session = ContextUtil.getCurrentUserSession(); if (session == null) { - throw restErrorUtil.createRESTException("UserSession cannot be null.", MessageEnums.OPER_NOT_ALLOWED_FOR_STATE); + throw restErrorUtil.createRESTException("UserSession cannot be null.", + MessageEnums.OPER_NOT_ALLOWED_FOR_STATE); } XXService xService = daoMgr.getXXService().getById(id); @@ -1345,11 +1472,13 @@ public RangerService getService(Long id) throws Exception { // services including KMS if (xService == null) { - throw restErrorUtil.createRESTException("Data Not Found for given Id", MessageEnums.DATA_NOT_FOUND, id, null, "readResource : No Object found with given id."); + throw restErrorUtil.createRESTException("Data Not Found for given Id", MessageEnums.DATA_NOT_FOUND, id, + null, "readResource : No Object found with given id."); } if (!bizUtil.hasAccess(xService, null)) { - throw restErrorUtil.createRESTException("Logged in user is not allowed to read service, id: " + id, MessageEnums.OPER_NO_PERMISSION); + throw restErrorUtil.createRESTException("Logged in user is not allowed to read service, id: " + id, + MessageEnums.OPER_NO_PERMISSION); } return svcService.getPopulatedViewObject(xService); @@ -1370,7 +1499,8 @@ public RangerService getServiceByName(String name) throws Exception { } if (!bizUtil.hasAccess(xService, null)) { - throw restErrorUtil.createRESTException("Logged in user is not allowed to read service, name: " + name, MessageEnums.OPER_NO_PERMISSION); + throw restErrorUtil.createRESTException("Logged in user is not allowed to read service, name: " + name, + MessageEnums.OPER_NO_PERMISSION); } } @@ -1389,7 +1519,9 @@ public RangerService getServiceByDisplayName(String displayName) { } if (!bizUtil.hasAccess(xService, null)) { - throw restErrorUtil.createRESTException("Logged in user is not allowed to read service, name: " + displayName, MessageEnums.OPER_NO_PERMISSION); + throw restErrorUtil.createRESTException( + "Logged in user is not allowed to read service, name: " + displayName, + MessageEnums.OPER_NO_PERMISSION); } } @@ -1425,8 +1557,8 @@ public RangerPolicy createDefaultPolicy(RangerPolicy policy) throws Exception { public RangerPolicy updatePolicy(RangerPolicy policy) throws Exception { LOG.debug("==> ServiceDBStore.updatePolicy({})", policy); - XXPolicy xxExisting = daoMgr.getXXPolicy().getById(policy.getId()); - RangerPolicy existing = policyService.getPopulatedViewObject(xxExisting); + XXPolicy xxExisting = daoMgr.getXXPolicy().getById(policy.getId()); + RangerPolicy existing = policyService.getPopulatedViewObject(xxExisting); if (existing == null) { throw new Exception("no policy exists with ID=" + policy.getId()); @@ -1445,21 +1577,24 @@ public RangerPolicy updatePolicy(RangerPolicy policy) throws Exception { } if (!StringUtils.equalsIgnoreCase(existing.getService(), policy.getService())) { - throw new Exception("policy id=" + policy.getId() + " already exists in service " + existing.getService() + ". It can not be moved to service " + policy.getService()); + throw new Exception("policy id=" + policy.getId() + " already exists in service " + existing.getService() + + ". It can not be moved to service " + policy.getService()); } boolean renamed = !StringUtils.equalsIgnoreCase(policy.getName(), existing.getName()); if (renamed) { - XXPolicy newNamePolicy = daoMgr.getXXPolicy().findByNameAndServiceIdAndZoneId(policy.getName(), service.getId(), xxExisting.getZoneId()); + XXPolicy newNamePolicy = daoMgr.getXXPolicy().findByNameAndServiceIdAndZoneId(policy.getName(), + service.getId(), xxExisting.getZoneId()); if (newNamePolicy != null) { - throw new Exception("another policy already exists with name '" + policy.getName() + "'. ID=" + newNamePolicy.getId()); + throw new Exception("another policy already exists with name '" + policy.getName() + "'. ID=" + + newNamePolicy.getId()); } } - List policyLabels = policy.getPolicyLabels(); - Set uniquePolicyLabels = new TreeSet<>(policyLabels); + List policyLabels = policy.getPolicyLabels(); + Set uniquePolicyLabels = new TreeSet<>(policyLabels); policy.setCreateTime(xxExisting.getCreateTime()); @@ -1479,18 +1614,20 @@ public RangerPolicy updatePolicy(RangerPolicy policy) throws Exception { deleteExistingPolicyLabel(policy); - policyRefUpdater.createNewPolMappingForRefTable(policy, newUpdPolicy, xServiceDef, bizUtil.getCreatePrincipalsIfAbsent(), true); + policyRefUpdater.createNewPolMappingForRefTable(policy, newUpdPolicy, xServiceDef, + bizUtil.getCreatePrincipalsIfAbsent(), true); createOrMapLabels(newUpdPolicy, uniquePolicyLabels); - RangerPolicy updPolicy = policyService.getPopulatedViewObject(newUpdPolicy); - boolean updateServiceInfoRoleVersion = false; + RangerPolicy updPolicy = policyService.getPopulatedViewObject(newUpdPolicy); + boolean updateServiceInfoRoleVersion = false; if (isSupportsRolesDownloadByService()) { updateServiceInfoRoleVersion = isRoleDownloadRequired(updPolicy, service); } - handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE, updPolicy, updateServiceInfoRoleVersion); + handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE, updPolicy, + updateServiceInfoRoleVersion); dataHistService.createObjectDataHistory(updPolicy, RangerDataHistService.ACTION_UPDATE); @@ -1516,7 +1653,8 @@ public void deletePolicy(RangerPolicy policy, RangerService service) throws Exce if (version == null) { version = 1L; - LOG.info("Found Version Value: `null`, so setting value of version to 1, While updating object, version should not be null."); + LOG.info( + "Found Version Value: `null`, so setting value of version to 1, While updating object, version should not be null."); } else { version = version + 1; } @@ -1529,7 +1667,8 @@ public void deletePolicy(RangerPolicy policy, RangerService service) throws Exce policyService.delete(policy); - createTransactionLog(policy, RangerPolicyService.OPERATION_IMPORT_DELETE_CONTEXT, RangerPolicyService.OPERATION_DELETE_CONTEXT); + createTransactionLog(policy, RangerPolicyService.OPERATION_IMPORT_DELETE_CONTEXT, + RangerPolicyService.OPERATION_DELETE_CONTEXT); handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE, policy, false); dataHistService.createObjectDataHistory(policy, RangerDataHistService.ACTION_DELETE); @@ -1547,8 +1686,8 @@ public void deletePolicy(RangerPolicy policy) throws Exception { throw new Exception("No such policy exists"); } - String policyName = policy.getName(); - RangerService service = getServiceByName(policy.getService()); + String policyName = policy.getName(); + RangerService service = getServiceByName(policy.getService()); if (service == null) { throw new Exception("service does not exist - name='" + policy.getService()); @@ -1558,14 +1697,16 @@ public void deletePolicy(RangerPolicy policy) throws Exception { if (version == null) { version = 1L; - LOG.info("Found Version Value: `null`, so setting value of version to 1, While updating object, version should not be null."); + LOG.info( + "Found Version Value: `null`, so setting value of version to 1, While updating object, version should not be null."); } else { version = version + 1; } policy.setVersion(version); - createTransactionLog(policy, RangerPolicyService.OPERATION_IMPORT_DELETE_CONTEXT, RangerPolicyService.OPERATION_DELETE_CONTEXT); + createTransactionLog(policy, RangerPolicyService.OPERATION_IMPORT_DELETE_CONTEXT, + RangerPolicyService.OPERATION_DELETE_CONTEXT); policyRefUpdater.cleanupRefTables(policy); @@ -1594,12 +1735,12 @@ public RangerPolicy getPolicy(Long id) throws Exception { public List getPolicies(SearchFilter filter) throws Exception { LOG.debug("==> ServiceDBStore.getPolicies()"); - boolean fetchTagPolicies = Boolean.parseBoolean(filter.getParam(SearchFilter.FETCH_TAG_POLICIES)); + boolean fetchTagPolicies = Boolean.parseBoolean(filter.getParam(SearchFilter.FETCH_TAG_POLICIES)); boolean fetchAllZonePolicies = Boolean.parseBoolean(filter.getParam(SearchFilter.FETCH_ZONE_UNZONE_POLICIES)); - String zoneName = filter.getParam(SearchFilter.ZONE_NAME); + String zoneName = filter.getParam(SearchFilter.ZONE_NAME); - List ret = new ArrayList<>(); - RangerPolicyList policyList = searchRangerPolicies(filter); + List ret = new ArrayList<>(); + RangerPolicyList policyList = searchRangerPolicies(filter); List resourcePolicies = policyList.getPolicies(); List tagPolicies; @@ -1637,7 +1778,7 @@ public List getPolicies(SearchFilter filter) throws Exception { public Long getPolicyId(final Long serviceId, final String policyName, final Long zoneId) { LOG.debug("==> ServiceDBStore.getPolicyId()"); - Long ret = null; + Long ret = null; XXPolicy xxPolicy = daoMgr.getXXPolicy().findByNameAndServiceIdAndZoneId(policyName, serviceId, zoneId); if (xxPolicy != null) { @@ -1650,9 +1791,11 @@ public Long getPolicyId(final Long serviceId, final String policyName, final Lon } @Override - public List getPoliciesByResourceSignature(String serviceName, String policySignature, Boolean isPolicyEnabled) throws Exception { - List xxPolicies = daoMgr.getXXPolicy().findByResourceSignatureByPolicyStatus(serviceName, policySignature, isPolicyEnabled); - List policies = new ArrayList<>(xxPolicies.size()); + public List getPoliciesByResourceSignature(String serviceName, String policySignature, + Boolean isPolicyEnabled) throws Exception { + List xxPolicies = daoMgr.getXXPolicy().findByResourceSignatureByPolicyStatus(serviceName, + policySignature, isPolicyEnabled); + List policies = new ArrayList<>(xxPolicies.size()); for (XXPolicy xxPolicy : xxPolicies) { RangerPolicy policy = policyService.getPopulatedViewObject(xxPolicy); @@ -1667,9 +1810,9 @@ public List getPoliciesByResourceSignature(String serviceName, Str public List getServicePolicies(Long serviceId, SearchFilter filter) throws Exception { LOG.debug("==> ServiceDBStore.getServicePolicies({})", serviceId); - String zoneName = filter.getParam(SearchFilter.FETCH_ZONE_NAME); - String denyCondition = filter.getParam(SearchFilter.FETCH_DENY_CONDITION); - XXService service = daoMgr.getXXService().getById(serviceId); + String zoneName = filter.getParam(SearchFilter.FETCH_ZONE_NAME); + String denyCondition = filter.getParam(SearchFilter.FETCH_DENY_CONDITION); + XXService service = daoMgr.getXXService().getById(serviceId); if (service == null) { throw new Exception("service does not exist - id='" + serviceId); @@ -1683,7 +1826,8 @@ public List getServicePolicies(Long serviceId, SearchFilter filter } } - LOG.debug("<== ServiceDBStore.getServicePolicies({}) : policy-count={}", serviceId, ret == null ? 0 : ret.size()); + LOG.debug("<== ServiceDBStore.getServicePolicies({}) : policy-count={}", serviceId, + ret == null ? 0 : ret.size()); return ret; } @@ -1692,8 +1836,8 @@ public List getServicePolicies(Long serviceId, SearchFilter filter public List getServicePolicies(String serviceName, SearchFilter filter) throws Exception { LOG.debug("==> ServiceDBStore.getServicePolicies({})", serviceName); - String zoneName = filter.getParam("zoneName"); - XXService service = daoMgr.getXXService().findByName(serviceName); + String zoneName = filter.getParam("zoneName"); + XXService service = daoMgr.getXXService().findByName(serviceName); if (service == null) { throw new Exception("service does not exist - name='" + serviceName); @@ -1711,11 +1855,13 @@ public List getServicePolicies(String serviceName, SearchFilter fi } @Override - public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion, boolean needsBackwardCompatibility) throws Exception { - LOG.debug("==> ServiceDBStore.getServicePoliciesIfUpdated({}, {}, {})", serviceName, lastKnownVersion, needsBackwardCompatibility); + public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion, + boolean needsBackwardCompatibility) throws Exception { + LOG.debug("==> ServiceDBStore.getServicePoliciesIfUpdated({}, {}, {})", serviceName, lastKnownVersion, + needsBackwardCompatibility); - ServicePolicies ret = null; - XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName); + ServicePolicies ret = null; + XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName); if (serviceDbObj == null) { throw new Exception("service does not exist. name=" + serviceName); @@ -1727,8 +1873,11 @@ public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long last LOG.warn("serviceVersionInfo does not exist. name={}", serviceName); } - if (lastKnownVersion == null || serviceVersionInfoDbObj == null || serviceVersionInfoDbObj.getPolicyVersion() == null || !lastKnownVersion.equals(serviceVersionInfoDbObj.getPolicyVersion())) { - ret = RangerServicePoliciesCache.getInstance().getServicePolicies(serviceName, serviceDbObj.getId(), lastKnownVersion, needsBackwardCompatibility, this); + if (lastKnownVersion == null || serviceVersionInfoDbObj == null + || serviceVersionInfoDbObj.getPolicyVersion() == null + || !lastKnownVersion.equals(serviceVersionInfoDbObj.getPolicyVersion())) { + ret = RangerServicePoliciesCache.getInstance().getServicePolicies(serviceName, serviceDbObj.getId(), + lastKnownVersion, needsBackwardCompatibility, this); } if (LOG.isDebugEnabled()) { @@ -1748,7 +1897,7 @@ public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long last ret.setTagPolicies(null); } else { - String tagServiceName = ret.getTagPolicies() != null ? ret.getTagPolicies().getServiceName() : null; + String tagServiceName = ret.getTagPolicies() != null ? ret.getTagPolicies().getServiceName() : null; boolean isTagServiceActive = isServiceActive(tagServiceName); if (!isTagServiceActive) { @@ -1756,8 +1905,11 @@ public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long last copy.setTagPolicies(null); - List copyPolicies = ret.getPolicies() != null ? new ArrayList<>(ret.getPolicies()) : null; - List copyPolicyDeltas = ret.getPolicyDeltas() != null ? new ArrayList<>(ret.getPolicyDeltas()) : null; + List copyPolicies = ret.getPolicies() != null ? new ArrayList<>(ret.getPolicies()) + : null; + List copyPolicyDeltas = ret.getPolicyDeltas() != null + ? new ArrayList<>(ret.getPolicyDeltas()) + : null; copy.setPolicies(copyPolicies); copy.setPolicyDeltas(copyPolicyDeltas); @@ -1766,8 +1918,9 @@ public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long last } } - Map securityZones = securityZoneStore.getSecurityZonesForService(serviceName); - ServicePolicies updatedServicePolicies = ret; + Map securityZones = securityZoneStore + .getSecurityZonesForService(serviceName); + ServicePolicies updatedServicePolicies = ret; if (MapUtils.isNotEmpty(securityZones)) { updatedServicePolicies = getUpdatedServicePoliciesForZones(ret, securityZones); @@ -1788,32 +1941,41 @@ public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long last } } - LOG.debug("getServicePoliciesIfUpdated({}, {}, {}): configs = {}", serviceName, lastKnownVersion, needsBackwardCompatibility, ret == null ? null : ret.getServiceConfig()); + LOG.debug("getServicePoliciesIfUpdated({}, {}, {}): configs = {}", serviceName, lastKnownVersion, + needsBackwardCompatibility, ret == null ? null : ret.getServiceConfig()); - LOG.debug("<== ServiceDBStore.getServicePoliciesIfUpdated({}, {}, {}): count={}", serviceName, lastKnownVersion, needsBackwardCompatibility, (ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()); + LOG.debug("<== ServiceDBStore.getServicePoliciesIfUpdated({}, {}, {}): count={}", serviceName, lastKnownVersion, + needsBackwardCompatibility, (ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size()); return ret; } @Override - public ServicePolicies getServicePolicyDeltasOrPolicies(String serviceName, Long lastKnownVersion) throws Exception { + public ServicePolicies getServicePolicyDeltasOrPolicies(String serviceName, Long lastKnownVersion) + throws Exception { boolean getOnlyDeltas = false; - LOG.debug("Support for incremental policy updates enabled using \"ranger.admin{}\" configuation parameter :[{}]", RangerCommonConstants.RANGER_ADMIN_SUFFIX_POLICY_DELTA, SUPPORTS_POLICY_DELTAS); + LOG.debug( + "Support for incremental policy updates enabled using \"ranger.admin{}\" configuation parameter :[{}]", + RangerCommonConstants.RANGER_ADMIN_SUFFIX_POLICY_DELTA, SUPPORTS_POLICY_DELTAS); return getServicePolicies(serviceName, lastKnownVersion, getOnlyDeltas, SUPPORTS_POLICY_DELTAS, Long.MAX_VALUE); } @Override - public ServicePolicies getServicePolicyDeltas(String serviceName, Long lastKnownVersion, Long cachedPolicyVersion) throws Exception { + public ServicePolicies getServicePolicyDeltas(String serviceName, Long lastKnownVersion, Long cachedPolicyVersion) + throws Exception { ServicePolicies ret = null; if (SUPPORTS_POLICY_DELTAS) { - LOG.debug("Support for incremental policy updates enabled using \"ranger.admin{}\" configuation parameter :[{}]", RangerCommonConstants.RANGER_ADMIN_SUFFIX_POLICY_DELTA, SUPPORTS_POLICY_DELTAS); + LOG.debug( + "Support for incremental policy updates enabled using \"ranger.admin{}\" configuation parameter :[{}]", + RangerCommonConstants.RANGER_ADMIN_SUFFIX_POLICY_DELTA, SUPPORTS_POLICY_DELTAS); ret = getServicePolicies(serviceName, lastKnownVersion, true, SUPPORTS_POLICY_DELTAS, cachedPolicyVersion); } - LOG.debug("<== ServiceDBStore.getServicePolicyDeltas({}, {}): ret = {}", serviceName, lastKnownVersion, ret == null ? ret : ret.getServiceConfig()); + LOG.debug("<== ServiceDBStore.getServicePolicyDeltas({}, {}): ret = {}", serviceName, lastKnownVersion, + ret == null ? ret : ret.getServiceConfig()); return ret; } @@ -1826,10 +1988,12 @@ public ServicePolicies getServicePolicies(String serviceName, Long lastKnownVers } public RangerPolicy getPolicyFromEventTime(String eventTime, Long policyId) { - XXDataHist xDataHist = daoMgr.getXXDataHist().findObjByEventTimeClassTypeAndId(eventTime, AppConstants.CLASS_TYPE_RANGER_POLICY, policyId); + XXDataHist xDataHist = daoMgr.getXXDataHist().findObjByEventTimeClassTypeAndId(eventTime, + AppConstants.CLASS_TYPE_RANGER_POLICY, policyId); if (xDataHist == null) { - String errMsg = "No policy history found for given policy ID: " + policyId + " and event time: " + eventTime; + String errMsg = "No policy history found for given policy ID: " + policyId + " and event time: " + + eventTime; LOG.error(errMsg); @@ -1884,7 +2048,7 @@ public long getPoliciesCount(final String serviceName) { public Map getServiceConfigForPlugin(Long serviceId) { LOG.debug("==> ServiceDBStore.getServiceConfigForPlugin({})", serviceId); - Map configs = new HashMap<>(); + Map configs = new HashMap<>(); List xxServiceConfigMaps = daoMgr.getXXServiceConfigMap().findByServiceId(serviceId); if (CollectionUtils.isNotEmpty(xxServiceConfigMaps)) { @@ -1895,7 +2059,8 @@ public Map getServiceConfigForPlugin(Long serviceId) { } } - Map rangerPluginsPrefixConfig = PropertiesUtil.getConfigMapWithPrefix(RANGER_PLUGINS_CONFIG_CONF_PREFIX); + Map rangerPluginsPrefixConfig = PropertiesUtil + .getConfigMapWithPrefix(RANGER_PLUGINS_CONFIG_CONF_PREFIX); if (MapUtils.isNotEmpty(rangerPluginsPrefixConfig)) { configs.putAll(rangerPluginsPrefixConfig); @@ -1916,12 +2081,15 @@ public List getPoliciesWithMetaAttributes(List polic policies.add(policyCopy); } - List policytimeMetaDataList = daoMgr.getXXPolicy().getMetaAttributesForPolicies(policies.stream().map(RangerPolicy::getId).collect(Collectors.toList())); + List policytimeMetaDataList = daoMgr.getXXPolicy().getMetaAttributesForPolicies( + policies.stream().map(RangerPolicy::getId).collect(Collectors.toList())); if (CollectionUtils.isNotEmpty(policytimeMetaDataList)) { Map> policyMap = policytimeMetaDataList.stream() - .filter(row -> row != null && row.length == 3 && row[0] != null && row[1] != null && row[2] != null) - .collect(Collectors.toMap(row -> (Long) row[0], row -> Arrays.asList((Date) row[1], (Date) row[2]))); + .filter(row -> row != null && row.length == 3 && row[0] != null && row[1] != null + && row[2] != null) + .collect(Collectors.toMap(row -> (Long) row[0], + row -> Arrays.asList((Date) row[1], (Date) row[2]))); for (RangerPolicy policy : policies) { List timeMetaData = policyMap.get(policy.getId()); @@ -1939,7 +2107,8 @@ public List getPoliciesWithMetaAttributes(List polic return policiesList; } - public void deletePolicies(Set policies, String serviceName, List deletedPolicyIds) throws Exception { + public void deletePolicies(Set policies, String serviceName, List deletedPolicyIds) + throws Exception { LOG.debug("==> ServiceDBStore.deletePolicies()"); if (policies == null) { @@ -1949,7 +2118,8 @@ public void deletePolicies(Set policies, String serviceName, List< RangerService service = getServiceByName(serviceName); if (service == null) { - throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, serviceName + ": service does not exist", true); + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, + serviceName + ": service does not exist", true); } boolean isBulkMode = RangerBizUtil.isBulkMode(); @@ -1978,7 +2148,8 @@ public void deletePolicies(Set policies, String serviceName, List< } } - LOG.debug("<== ServiceDBStore.deletePolicies(policyCount={}): deletedCount={}", policies.size(), deletedPolicyIds.size()); + LOG.debug("<== ServiceDBStore.deletePolicies(policyCount={}): deletedCount={}", policies.size(), + deletedPolicyIds.size()); } @PostConstruct @@ -1987,7 +2158,8 @@ public void initStore() { config = RangerAdminConfig.getInstance(); - String nullSafeSupplier = config.get("ranger.admin.null_safe.supplier", RangerBaseModelObject.NULL_SAFE_SUPPLIER_V2); + String nullSafeSupplier = config.get("ranger.admin.null_safe.supplier", + RangerBaseModelObject.NULL_SAFE_SUPPLIER_V2); LOG.info("ranger.admin.null_safe.supplier={}", nullSafeSupplier); @@ -1996,19 +2168,29 @@ public void initStore() { if (!legacyServiceDefsInitDone) { synchronized (ServiceDBStore.class) { if (!legacyServiceDefsInitDone) { - SUPPORTS_POLICY_DELTAS = config.getBoolean("ranger.admin" + RangerCommonConstants.RANGER_ADMIN_SUFFIX_POLICY_DELTA, RangerCommonConstants.RANGER_ADMIN_SUFFIX_POLICY_DELTA_DEFAULT); - RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.delta.retention.time.in.days", 7); + SUPPORTS_POLICY_DELTAS = config.getBoolean( + "ranger.admin" + RangerCommonConstants.RANGER_ADMIN_SUFFIX_POLICY_DELTA, + RangerCommonConstants.RANGER_ADMIN_SUFFIX_POLICY_DELTA_DEFAULT); + RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.delta.retention.time.in.days", 7); TAG_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.tag.delta.retention.time.in.days", 3); - SUPPORTS_PURGE_LOGIN_RECORDS = config.getBoolean("ranger.admin.init.purge.login_records", false); - SUPPORTS_PURGE_TRANSACTION_RECORDS = config.getBoolean("ranger.admin.init.purge.transaction_records", false); - SUPPORTS_PURGE_POLICY_EXPORT_LOGS = config.getBoolean("ranger.admin.init.purge.policy_export_logs", false); - LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.init.purge.login_records.retention.days", 0); - TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.init.purge.transaction_records.retention.days", 0); - POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.init.purge.policy_export_logs.retention.days", 0); - - isRolesDownloadedByService = config.getBoolean("ranger.support.for.service.specific.role.download", false); - SUPPORTS_IN_PLACE_POLICY_UPDATES = SUPPORTS_POLICY_DELTAS && config.getBoolean("ranger.admin" + RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES, RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES_DEFAULT); + SUPPORTS_PURGE_LOGIN_RECORDS = config.getBoolean("ranger.admin.init.purge.login_records", false); + SUPPORTS_PURGE_TRANSACTION_RECORDS = config + .getBoolean("ranger.admin.init.purge.transaction_records", false); + SUPPORTS_PURGE_POLICY_EXPORT_LOGS = config.getBoolean("ranger.admin.init.purge.policy_export_logs", + false); + LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS = config + .getInt("ranger.admin.init.purge.login_records.retention.days", 0); + TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS = config + .getInt("ranger.admin.init.purge.transaction_records.retention.days", 0); + POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS = config + .getInt("ranger.admin.init.purge.policy_export_logs.retention.days", 0); + + isRolesDownloadedByService = config.getBoolean("ranger.support.for.service.specific.role.download", + false); + SUPPORTS_IN_PLACE_POLICY_UPDATES = SUPPORTS_POLICY_DELTAS && config.getBoolean( + "ranger.admin" + RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES, + RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES_DEFAULT); LOG.info("SUPPORTS_POLICY_DELTAS={}", SUPPORTS_POLICY_DELTAS); LOG.info("RETENTION_PERIOD_IN_DAYS={}", RETENTION_PERIOD_IN_DAYS); @@ -2016,14 +2198,16 @@ public void initStore() { LOG.info("SUPPORTS_PURGE_LOGIN_RECORDS={}", SUPPORTS_PURGE_LOGIN_RECORDS); LOG.info("LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS={}", LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS); LOG.info("SUPPORTS_PURGE_TRANSACTION_RECORDS={}", SUPPORTS_PURGE_TRANSACTION_RECORDS); - LOG.info("TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS={}", TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS); + LOG.info("TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS={}", + TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS); LOG.info("SUPPORTS_PURGE_POLICY_EXPORT_LOGS={}", SUPPORTS_PURGE_POLICY_EXPORT_LOGS); - LOG.info("POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS={}", POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS); + LOG.info("POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS={}", + POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS); LOG.info("isRolesDownloadedByService={}", isRolesDownloadedByService); LOG.info("SUPPORTS_IN_PLACE_POLICY_UPDATES={}", SUPPORTS_IN_PLACE_POLICY_UPDATES); - TransactionTemplate txTemplate = new TransactionTemplate(txManager); - final ServiceDBStore dbStore = this; + TransactionTemplate txTemplate = new TransactionTemplate(txManager); + final ServiceDBStore dbStore = this; predicateUtil = new ServicePredicateUtil(dbStore); @@ -2032,8 +2216,10 @@ public void initStore() { EmbeddedServiceDefsUtil.instance().init(dbStore); getServiceUpgraded(); createGenericUsers(); - resetPolicyUpdateLog(RETENTION_PERIOD_IN_DAYS, RangerPolicyDelta.CHANGE_TYPE_RANGER_ADMIN_START); - resetTagUpdateLog(TAG_RETENTION_PERIOD_IN_DAYS, ServiceTags.TagsChangeType.RANGER_ADMIN_START); + resetPolicyUpdateLog(RETENTION_PERIOD_IN_DAYS, + RangerPolicyDelta.CHANGE_TYPE_RANGER_ADMIN_START); + resetTagUpdateLog(TAG_RETENTION_PERIOD_IN_DAYS, + ServiceTags.TagsChangeType.RANGER_ADMIN_START); List purgeResults = new ArrayList<>(); @@ -2072,7 +2258,8 @@ public void deleteXXAccessTypeDef(XXAccessTypeDef xAccess) { daoMgr.getXXAccessTypeDefGrants().remove(atdGrant); } - List policyRefAccessTypeList = daoMgr.getXXPolicyRefAccessType().findByAccessTypeDefId(xAccess.getId()); + List policyRefAccessTypeList = daoMgr.getXXPolicyRefAccessType() + .findByAccessTypeDefId(xAccess.getId()); for (XXPolicyRefAccessType xxPolicyRefAccessType : policyRefAccessTypeList) { daoMgr.getXXPolicyRefAccessType().remove(xxPolicyRefAccessType); @@ -2088,7 +2275,8 @@ public void deleteXXResourceDef(XXResourceDef xRes) { deleteXXResourceDef(childRes); } - List xxPolicyRefResources = daoMgr.getXXPolicyRefResource().findByResourceDefID(xRes.getId()); + List xxPolicyRefResources = daoMgr.getXXPolicyRefResource() + .findByResourceDefID(xRes.getId()); for (XXPolicyRefResource xPolRefRes : xxPolicyRefResources) { daoMgr.getXXPolicyRefResource().remove(xPolRefRes); @@ -2108,7 +2296,9 @@ public PList getPaginatedServiceDefs(SearchFilter filter) thro LOG.debug("==> ServiceDBStore.getPaginatedServiceDefs({})", filter); - return new PList<>(svcDefList.getServiceDefs(), svcDefList.getStartIndex(), svcDefList.getPageSize(), svcDefList.getTotalCount(), svcDefList.getResultSize(), svcDefList.getSortType(), svcDefList.getSortBy()); + return new PList<>(svcDefList.getServiceDefs(), svcDefList.getStartIndex(), svcDefList.getPageSize(), + svcDefList.getTotalCount(), svcDefList.getResultSize(), svcDefList.getSortType(), + svcDefList.getSortBy()); } public PList getPaginatedServices(SearchFilter filter) throws Exception { @@ -2122,7 +2312,9 @@ public PList getPaginatedServices(SearchFilter filter) throws Exc LOG.debug("<== ServiceDBStore.getPaginatedServices()"); - return new PList<>(serviceList.getServices(), serviceList.getStartIndex(), serviceList.getPageSize(), serviceList.getTotalCount(), serviceList.getResultSize(), serviceList.getSortType(), serviceList.getSortBy()); + return new PList<>(serviceList.getServices(), serviceList.getStartIndex(), serviceList.getPageSize(), + serviceList.getTotalCount(), serviceList.getResultSize(), serviceList.getSortType(), + serviceList.getSortBy()); } public PList getPaginatedPolicies(SearchFilter filter) { @@ -2138,7 +2330,9 @@ public PList getPaginatedPolicies(SearchFilter filter) { LOG.debug("<== ServiceDBStore.getPaginatedPolicies({}): count={}", filter, policyList.getListSize()); - return new PList<>(policyList.getPolicies(), policyList.getStartIndex(), policyList.getPageSize(), policyList.getTotalCount(), policyList.getResultSize(), policyList.getSortType(), policyList.getSortBy()); + return new PList<>(policyList.getPolicies(), policyList.getStartIndex(), policyList.getPageSize(), + policyList.getTotalCount(), policyList.getResultSize(), policyList.getSortType(), + policyList.getSortBy()); } public PList getPaginatedServicePolicies(Long serviceId, SearchFilter filter) throws Exception { @@ -2168,7 +2362,8 @@ public PList getPaginatedServicePolicies(String serviceName, Searc PList ret = getPaginatedPolicies(filter); - LOG.debug("<== ServiceDBStore.getPaginatedServicePolicies({}): count={}", serviceName, (ret == null) ? 0 : ret.getListSize()); + LOG.debug("<== ServiceDBStore.getPaginatedServicePolicies({}): count={}", serviceName, + (ret == null) ? 0 : ret.getListSize()); return ret; } @@ -2180,17 +2375,20 @@ public Long getServicePolicyVersion(String serviceName) { return serviceVersionInfoDbObj != null ? serviceVersionInfoDbObj.getPolicyVersion() : null; } - // when a service-def is updated, the updated service-def should be made available to plugins - // this is achieved by incrementing policyVersion of all services of this service-def + // when a service-def is updated, the updated service-def should be made + // available to plugins + // this is achieved by incrementing policyVersion of all services of this + // service-def protected void updateServicesForServiceDefUpdate(RangerServiceDef serviceDef) { if (serviceDef == null) { return; } - final RangerDaoManager daoManager = daoMgr; - boolean isTagServiceDef = StringUtils.equals(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME); - XXServiceDao serviceDao = daoMgr.getXXService(); - List services = serviceDao.findByServiceDefId(serviceDef.getId()); + final RangerDaoManager daoManager = daoMgr; + boolean isTagServiceDef = StringUtils.equals(serviceDef.getName(), + EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME); + XXServiceDao serviceDao = daoMgr.getXXService(); + List services = serviceDao.findByServiceDefId(serviceDef.getId()); if (CollectionUtils.isNotEmpty(services)) { for (XXService service : services) { @@ -2199,20 +2397,23 @@ protected void updateServicesForServiceDefUpdate(RangerServiceDef serviceDef) { if (CollectionUtils.isNotEmpty(referringServices)) { for (XXService referringService : referringServices) { - final Long referringServiceId = referringService.getId(); + final Long referringServiceId = referringService.getId(); final VERSION_TYPE tagServiceVersionType = VERSION_TYPE.POLICY_VERSION; - Runnable tagServiceVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, tagServiceVersionType, RangerPolicyDelta.CHANGE_TYPE_SERVICE_DEF_CHANGE); + Runnable tagServiceVersionUpdater = new ServiceVersionUpdater(daoManager, + referringServiceId, tagServiceVersionType, + RangerPolicyDelta.CHANGE_TYPE_SERVICE_DEF_CHANGE); transactionSynchronizationAdapter.executeOnTransactionCommit(tagServiceVersionUpdater); } } } - final Long serviceId = service.getId(); + final Long serviceId = service.getId(); final VERSION_TYPE versionType = VERSION_TYPE.POLICY_VERSION; - Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType, RangerPolicyDelta.CHANGE_TYPE_SERVICE_DEF_CHANGE); + Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType, + RangerPolicyDelta.CHANGE_TYPE_SERVICE_DEF_CHANGE); transactionSynchronizationAdapter.executeOnTransactionCommit(serviceVersionUpdater); } @@ -2250,7 +2451,7 @@ public RangerPolicy createPolicy(RangerPolicy policy, boolean createPrincipalsIf throw new Exception("service-def does not exist - name=" + service.getType()); } - Long zoneId = RangerSecurityZone.RANGER_UNZONED_SECURITY_ZONE_ID; + Long zoneId = RangerSecurityZone.RANGER_UNZONED_SECURITY_ZONE_ID; String zoneName = policy.getZoneName(); if (StringUtils.isNotEmpty(zoneName)) { @@ -2263,14 +2464,16 @@ public RangerPolicy createPolicy(RangerPolicy policy, boolean createPrincipalsIf } } - XXPolicy existing = daoMgr.getXXPolicy().findByNameAndServiceIdAndZoneId(policy.getName(), service.getId(), zoneId); + XXPolicy existing = daoMgr.getXXPolicy().findByNameAndServiceIdAndZoneId(policy.getName(), service.getId(), + zoneId); if (existing != null) { - throw new Exception("policy already exists: ServiceName=" + policy.getService() + "; PolicyName=" + policy.getName() + ". ID=" + existing.getId()); + throw new Exception("policy already exists: ServiceName=" + policy.getService() + "; PolicyName=" + + policy.getName() + ". ID=" + existing.getId()); } - List policyLabels = policy.getPolicyLabels(); - Set uniquePolicyLabels = new TreeSet<>(policyLabels); + List policyLabels = policy.getPolicyLabels(); + Set uniquePolicyLabels = new TreeSet<>(policyLabels); policy.setVersion(1L); @@ -2293,22 +2496,25 @@ public RangerPolicy createPolicy(RangerPolicy policy, boolean createPrincipalsIf XXPolicy xCreatedPolicy = daoMgr.getXXPolicy().getById(policy.getId()); - policyRefUpdater.createNewPolMappingForRefTable(policy, xCreatedPolicy, xServiceDef, createPrincipalsIfAbsent, false); + policyRefUpdater.createNewPolMappingForRefTable(policy, xCreatedPolicy, xServiceDef, createPrincipalsIfAbsent, + false); createOrMapLabels(xCreatedPolicy, uniquePolicyLabels); - RangerPolicy createdPolicy = policyService.getPopulatedViewObject(xCreatedPolicy); - boolean updateServiceInfoRoleVersion = false; + RangerPolicy createdPolicy = policyService.getPopulatedViewObject(xCreatedPolicy); + boolean updateServiceInfoRoleVersion = false; if (isSupportsRolesDownloadByService()) { updateServiceInfoRoleVersion = isRoleDownloadRequired(createdPolicy, service); } - handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE, createdPolicy, updateServiceInfoRoleVersion); + handlePolicyUpdate(service, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE, createdPolicy, + updateServiceInfoRoleVersion); dataHistService.createObjectDataHistory(createdPolicy, RangerDataHistService.ACTION_CREATE); - createTransactionLog(createdPolicy, RangerPolicyService.OPERATION_IMPORT_CREATE_CONTEXT, RangerPolicyService.OPERATION_CREATE_CONTEXT); + createTransactionLog(createdPolicy, RangerPolicyService.OPERATION_IMPORT_CREATE_CONTEXT, + RangerPolicyService.OPERATION_CREATE_CONTEXT); return createdPolicy; } @@ -2317,9 +2523,10 @@ public void createOrMapLabels(XXPolicy xPolicy, Set uniquePolicyLabels) LOG.debug("==> ServiceDBStore.createOrMapLabels()"); for (String policyLabel : uniquePolicyLabels) { - //check and create new label If does not exist + // check and create new label If does not exist if (StringUtils.isNotEmpty(policyLabel)) { - transactionSynchronizationAdapter.executeOnTransactionCommit(new AssociatePolicyLabel(policyLabel, xPolicy)); + transactionSynchronizationAdapter + .executeOnTransactionCommit(new AssociatePolicyLabel(policyLabel, xPolicy)); } } @@ -2330,7 +2537,8 @@ public RangerPolicy getPolicy(String guid, String serviceName, String zoneName) RangerPolicy ret = null; if (StringUtils.isNotBlank(guid)) { - XXPolicy xPolicy = daoMgr.getXXPolicy().findPolicyByGUIDAndServiceNameAndZoneName(guid, serviceName, zoneName); + XXPolicy xPolicy = daoMgr.getXXPolicy().findPolicyByGUIDAndServiceNameAndZoneName(guid, serviceName, + zoneName); if (xPolicy != null) { ret = policyService.getPopulatedViewObject(xPolicy); @@ -2343,7 +2551,7 @@ public RangerPolicy getPolicy(String guid, String serviceName, String zoneName) public void getPoliciesInExcel(List policies, HttpServletResponse response) throws Exception { LOG.debug("==> ServiceDBStore.getPoliciesInExcel()"); - String timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()); + String timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()); String excelFileName = "Ranger_Policies_" + timeStamp + ".xls"; writeExcel(policies, excelFileName, response); @@ -2352,14 +2560,14 @@ public void getPoliciesInExcel(List policies, HttpServletResponse public void getPoliciesInCSV(List policies, HttpServletResponse response) throws Exception { LOG.debug("==> ServiceDBStore.getPoliciesInCSV()"); - ServletOutputStream out = null; - String csvfilename = null; + ServletOutputStream out = null; + String csvfilename = null; try { String timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()); csvfilename = "Ranger_Policies_" + timeStamp + ".csv"; - out = response.getOutputStream(); + out = response.getOutputStream(); StringBuilder sb = writeCSV(policies, csvfilename, response); @@ -2380,7 +2588,8 @@ public void getPoliciesInCSV(List policies, HttpServletResponse re } } - public void getObjectInJson(List objList, HttpServletResponse response, JSON_FILE_NAME_TYPE type) throws Exception { + public void getObjectInJson(List objList, HttpServletResponse response, JSON_FILE_NAME_TYPE type) + throws Exception { LOG.debug("==> ServiceDBStore.getObjectInJson()"); String timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()); @@ -2451,8 +2660,8 @@ public void deleteZonePolicies(Collection serviceNames, Long zoneId) thr XXPolicyDao policyDao = daoMgr.getXXPolicy(); for (String serviceName : serviceNames) { - RangerService service = getServiceByName(serviceName); - List policyIds = policyDao.findPolicyIdsByServiceNameAndZoneId(serviceName, zoneId); + RangerService service = getServiceByName(serviceName); + List policyIds = policyDao.findPolicyIdsByServiceNameAndZoneId(serviceName, zoneId); if (CollectionUtils.isNotEmpty(policyIds)) { List rangerPolicyList = new ArrayList<>(); @@ -2480,7 +2689,8 @@ public void deleteZonePolicies(Collection serviceNames, Long zoneId) thr } public VXString getPolicyVersionList(Long policyId) { - List versionList = daoMgr.getXXDataHist().getVersionListOfObject(policyId, AppConstants.CLASS_TYPE_RANGER_POLICY); + List versionList = daoMgr.getXXDataHist().getVersionListOfObject(policyId, + AppConstants.CLASS_TYPE_RANGER_POLICY); VXString vXString = new VXString(); @@ -2490,7 +2700,8 @@ public VXString getPolicyVersionList(Long policyId) { } public RangerPolicy getPolicyForVersionNumber(Long policyId, int versionNo) { - XXDataHist xDataHist = daoMgr.getXXDataHist().findObjectByVersionNumber(policyId, AppConstants.CLASS_TYPE_RANGER_POLICY, versionNo); + XXDataHist xDataHist = daoMgr.getXXDataHist().findObjectByVersionNumber(policyId, + AppConstants.CLASS_TYPE_RANGER_POLICY, versionNo); if (xDataHist == null) { throw restErrorUtil.createRESTException("No Policy found for given version.", MessageEnums.DATA_NOT_FOUND); @@ -2503,9 +2714,9 @@ public RangerPolicy getPolicyForVersionNumber(Long policyId, int versionNo) { public Map getMetaDataInfo() { Map metaDataInfo = new LinkedHashMap<>(); - UserSessionBase usb = ContextUtil.getCurrentUserSession(); - String userId = usb != null ? usb.getLoginId() : null; - DateFormat formatter = new SimpleDateFormat("MMM dd, yyyy h:mm:ss a"); + UserSessionBase usb = ContextUtil.getCurrentUserSession(); + String userId = usb != null ? usb.getLoginId() : null; + DateFormat formatter = new SimpleDateFormat("MMM dd, yyyy h:mm:ss a"); metaDataInfo.put(HOSTNAME, LOCAL_HOSTNAME); metaDataInfo.put(USER_NAME, userId); @@ -2518,8 +2729,8 @@ public Map getMetaDataInfo() { public Map getMapFromInputStream(InputStream mapStream) throws IOException { LOG.debug("==> ServiceDBStore.getMapFromInputStream()"); - Map inputMap = new LinkedHashMap<>(); - String inputMapString = IOUtils.toString(mapStream); + Map inputMap = new LinkedHashMap<>(); + String inputMapString = IOUtils.toString(mapStream); if (StringUtils.isNotEmpty(inputMapString)) { inputMap = jsonUtil.jsonToMap(inputMapString); @@ -2537,16 +2748,20 @@ public Map getMapFromInputStream(InputStream mapStream) throws I } public Map setPolicyMapKeyValue(Map policiesMap, RangerPolicy policy) { - if (StringUtils.isNotEmpty(policy.getName().trim()) && StringUtils.isNotEmpty(policy.getService().trim()) && StringUtils.isNotEmpty(policy.getResources().toString().trim())) { - policiesMap.put(policy.getName().trim() + " " + policy.getService().trim() + " " + policy.getResources().toString().trim() + " " + policy.getZoneName(), policy); + if (StringUtils.isNotEmpty(policy.getName().trim()) && StringUtils.isNotEmpty(policy.getService().trim()) + && StringUtils.isNotEmpty(policy.getResources().toString().trim())) { + policiesMap.put(policy.getName().trim() + " " + policy.getService().trim() + " " + + policy.getResources().toString().trim() + " " + policy.getZoneName(), policy); } else if (StringUtils.isEmpty(policy.getName().trim()) && StringUtils.isNotEmpty(policy.getService().trim())) { LOG.error("Policy Name is not provided for service : {}", policy.getService().trim()); - throw restErrorUtil.createRESTException("Policy Name is not provided for service : " + policy.getService().trim()); + throw restErrorUtil + .createRESTException("Policy Name is not provided for service : " + policy.getService().trim()); } else if (StringUtils.isNotEmpty(policy.getName().trim()) && StringUtils.isEmpty(policy.getService().trim())) { LOG.error("Service Name is not provided for policy : {}", policy.getName().trim()); - throw restErrorUtil.createRESTException("Service Name is not provided for policy : " + policy.getName().trim()); + throw restErrorUtil + .createRESTException("Service Name is not provided for policy : " + policy.getName().trim()); } else { LOG.error("Service Name or Policy Name is not provided!!"); @@ -2556,7 +2771,9 @@ public Map setPolicyMapKeyValue(Map return policiesMap; } - public Map createPolicyMap(Map zoneMappingMap, List sourceZones, String destinationZoneName, Map servicesMappingMap, List sourceServices, List destinationServices, RangerPolicy policy, Map policiesMap) { + public Map createPolicyMap(Map zoneMappingMap, List sourceZones, + String destinationZoneName, Map servicesMappingMap, List sourceServices, + List destinationServices, RangerPolicy policy, Map policiesMap) { if (!CollectionUtils.sizeIsEmpty(zoneMappingMap)) { policy.setZoneName(destinationZoneName); // set destination zone name in policy. } @@ -2599,7 +2816,8 @@ public void resetPolicyUpdateLog(int retentionInDays, Integer policyChangeType) if (CollectionUtils.isNotEmpty(allServiceIds)) { for (Long serviceId : allServiceIds) { - ServiceVersionUpdater updater = new ServiceVersionUpdater(daoMgr, serviceId, VERSION_TYPE.POLICY_VERSION, null, policyChangeType, null); + ServiceVersionUpdater updater = new ServiceVersionUpdater(daoMgr, serviceId, + VERSION_TYPE.POLICY_VERSION, null, policyChangeType, null); persistVersionChange(updater); } @@ -2617,7 +2835,8 @@ public void resetTagUpdateLog(int retentionInDays, ServiceTags.TagsChangeType ta if (CollectionUtils.isNotEmpty(allServiceIds)) { for (Long serviceId : allServiceIds) { - ServiceVersionUpdater updater = new ServiceVersionUpdater(daoMgr, serviceId, VERSION_TYPE.TAG_VERSION, tagChangeType, null, null); + ServiceVersionUpdater updater = new ServiceVersionUpdater(daoMgr, serviceId, VERSION_TYPE.TAG_VERSION, + tagChangeType, null, null); persistVersionChange(updater); } @@ -2630,13 +2849,15 @@ public void removeAuthSessions(int retentionInDays, List resu LOG.debug("==> removeAuthSessions({})", retentionInDays); if (retentionInDays > 0) { - XXAuthSessionDao dao = daoMgr.getXXAuthSession(); - long rowsCount = dao.getAllCount(); - long rowsDeleted = dao.deleteOlderThan(retentionInDays); + XXAuthSessionDao dao = daoMgr.getXXAuthSession(); + long rowsCount = dao.getAllCount(); + long rowsDeleted = dao.deleteOlderThan(retentionInDays); LOG.info("Deleted {} records from x_auth_sess that are older than {} days", rowsDeleted, retentionInDays); - svcService.createTransactionLog(new XXTrxLogV2(AppConstants.CLASS_TYPE_AUTH_SESS, null, null, "Deleted Auth Session records"), "Records count", "Total Records : " + rowsCount, "Deleted Records : " + rowsDeleted); + svcService.createTransactionLog( + new XXTrxLogV2(AppConstants.CLASS_TYPE_AUTH_SESS, null, null, "Deleted Auth Session records"), + "Records count", "Total Records : " + rowsCount, "Deleted Records : " + rowsDeleted); result.add(new RangerPurgeResult(ServiceREST.PURGE_RECORD_TYPE_LOGIN_LOGS, rowsCount, rowsDeleted)); } @@ -2648,13 +2869,15 @@ public void removeTransactionLogs(int retentionInDays, List r LOG.debug("==> removeTransactionLogs({})", retentionInDays); if (retentionInDays > 0) { - XXTrxLogV2Dao dao = daoMgr.getXXTrxLogV2(); - long rowsCount = dao.getAllCount(); - long rowsDeleted = dao.deleteOlderThan(retentionInDays); + XXTrxLogV2Dao dao = daoMgr.getXXTrxLogV2(); + long rowsCount = dao.getAllCount(); + long rowsDeleted = dao.deleteOlderThan(retentionInDays); LOG.info("Deleted {} records from x_trx_log that are older than {} days", rowsDeleted, retentionInDays); - svcService.createTransactionLog(new XXTrxLogV2(AppConstants.CLASS_TYPE_TRX_LOG, null, null, "Deleted Transaction records"), "Records count", "Total Records : " + rowsCount, "Deleted Records : " + rowsDeleted); + svcService.createTransactionLog( + new XXTrxLogV2(AppConstants.CLASS_TYPE_TRX_LOG, null, null, "Deleted Transaction records"), + "Records count", "Total Records : " + rowsCount, "Deleted Records : " + rowsDeleted); result.add(new RangerPurgeResult(ServiceREST.PURGE_RECORD_TYPE_TRX_LOGS, rowsCount, rowsDeleted)); } @@ -2666,13 +2889,17 @@ public void removePolicyExportLogs(int retentionInDays, List LOG.debug("==> removePolicyExportLogs({})", retentionInDays); if (retentionInDays > 0) { - XXPolicyExportAuditDao dao = daoMgr.getXXPolicyExportAudit(); - long rowsCount = dao.getAllCount(); - long rowsDeleted = dao.deleteOlderThan(retentionInDays); + XXPolicyExportAuditDao dao = daoMgr.getXXPolicyExportAudit(); + long rowsCount = dao.getAllCount(); + long rowsDeleted = dao.deleteOlderThan(retentionInDays); - LOG.info("Deleted {} records from x_policy_export_audit that are older than {} days", rowsDeleted, retentionInDays); + LOG.info("Deleted {} records from x_policy_export_audit that are older than {} days", rowsDeleted, + retentionInDays); - policyService.createTransactionLog(new XXTrxLogV2(AppConstants.CLASS_TYPE_XA_POLICY_EXPORT_AUDIT, null, null, "Deleted policy export audit records"), "Records count", "Total Records : " + rowsCount, "Deleted Records : " + rowsDeleted); + policyService.createTransactionLog( + new XXTrxLogV2(AppConstants.CLASS_TYPE_XA_POLICY_EXPORT_AUDIT, null, null, + "Deleted policy export audit records"), + "Records count", "Total Records : " + rowsCount, "Deleted Records : " + rowsDeleted); result.add(new RangerPurgeResult(ServiceREST.PURGE_RECORD_TYPE_POLICY_EXPORT_LOGS, rowsCount, rowsDeleted)); } @@ -2683,9 +2910,10 @@ public void removePolicyExportLogs(int retentionInDays, List public List getPolicyLabels(SearchFilter searchFilter) { LOG.debug("==> ServiceDBStore.getPolicyLabels()"); - VXPolicyLabelList vxPolicyLabelList = new VXPolicyLabelList(); - List xPolList = policyLabelsService.searchResources(searchFilter, policyLabelsService.searchFields, policyLabelsService.sortFields, vxPolicyLabelList); - List result = new ArrayList<>(); + VXPolicyLabelList vxPolicyLabelList = new VXPolicyLabelList(); + List xPolList = policyLabelsService.searchResources(searchFilter, + policyLabelsService.searchFields, policyLabelsService.sortFields, vxPolicyLabelList); + List result = new ArrayList<>(); for (XXPolicyLabel xPolicyLabel : xPolList) { result.add(xPolicyLabel.getPolicyLabel()); @@ -2697,12 +2925,15 @@ public List getPolicyLabels(SearchFilter searchFilter) { } /** - * This method returns {@linkplain java.util.Map map} representing policy count for each service Definition, - * filtered by policy type, if policy type is not valid (null or less than zero) default policy type will + * This method returns {@linkplain java.util.Map map} representing policy count + * for each service Definition, + * filtered by policy type, if policy type is not valid (null or less than zero) + * default policy type will * be used (ie Resource Access) * * @param policyType - * @return {@linkplain java.util.Map map} representing policy count for each service Definition + * @return {@linkplain java.util.Map map} representing policy count for each + * service Definition */ public Map getPolicyCountByTypeAndServiceType(Integer policyType) { int type = 0; @@ -2746,10 +2977,11 @@ public String getMetricByType(final METRIC_TYPE metricType) throws Exception { } public boolean isServiceAdminUser(String serviceName, String userName) { - boolean ret = false; - XXServiceConfigMapDao svcCfgMapDao = daoMgr.getXXServiceConfigMap(); - XXServiceConfigMap cfgSvcAdminUsers = svcCfgMapDao.findByServiceNameAndConfigKey(serviceName, SERVICE_ADMIN_USERS); - String svcAdminUsers = cfgSvcAdminUsers != null ? cfgSvcAdminUsers.getConfigvalue() : null; + boolean ret = false; + XXServiceConfigMapDao svcCfgMapDao = daoMgr.getXXServiceConfigMap(); + XXServiceConfigMap cfgSvcAdminUsers = svcCfgMapDao.findByServiceNameAndConfigKey(serviceName, + SERVICE_ADMIN_USERS); + String svcAdminUsers = cfgSvcAdminUsers != null ? cfgSvcAdminUsers.getConfigvalue() : null; if (svcAdminUsers != null) { for (String svcAdminUser : svcAdminUsers.split(",")) { @@ -2761,8 +2993,9 @@ public boolean isServiceAdminUser(String serviceName, String userName) { } if (!ret) { - XXServiceConfigMap cfgSvcAdminGroups = svcCfgMapDao.findByServiceNameAndConfigKey(serviceName, SERVICE_ADMIN_GROUPS); - String svcAdminGroups = cfgSvcAdminGroups != null ? cfgSvcAdminGroups.getConfigvalue() : null; + XXServiceConfigMap cfgSvcAdminGroups = svcCfgMapDao.findByServiceNameAndConfigKey(serviceName, + SERVICE_ADMIN_GROUPS); + String svcAdminGroups = cfgSvcAdminGroups != null ? cfgSvcAdminGroups.getConfigvalue() : null; if (StringUtils.isNotBlank(svcAdminGroups)) { Set userGroups = xUserMgr.getGroupsForUser(userName); @@ -2783,7 +3016,8 @@ public boolean isServiceAdminUser(String serviceName, String userName) { } public void updateServiceAuditConfig(String searchUsrGrpRoleName, REMOVE_REF_TYPE removeRefType) { - LOG.debug("===> ServiceDBStore.updateServiceAuditConfig( searchUsrGrpRoleName : {} removeRefType : {})", searchUsrGrpRoleName, removeRefType); + LOG.debug("===> ServiceDBStore.updateServiceAuditConfig( searchUsrGrpRoleName : {} removeRefType : {})", + searchUsrGrpRoleName, removeRefType); List configMapToBeModified = getAuditFiltersServiceConfigByName(searchUsrGrpRoleName); @@ -2793,12 +3027,12 @@ public void updateServiceAuditConfig(String searchUsrGrpRoleName, REMOVE_REF_TYP if (StringUtils.isNotBlank(jsonStr)) { List auditFilters = JsonUtils.jsonToAuditFilterList(jsonStr); - int filterCount = auditFilters != null ? auditFilters.size() : 0; + int filterCount = auditFilters != null ? auditFilters.size() : 0; if (filterCount > 0) { - String userName = null; + String userName = null; String groupName = null; - String roleName = null; + String roleName = null; if (removeRefType == REMOVE_REF_TYPE.USER) { userName = searchUsrGrpRoleName; @@ -2810,18 +3044,21 @@ public void updateServiceAuditConfig(String searchUsrGrpRoleName, REMOVE_REF_TYP removeUserGroupRoleReferences(auditFilters, userName, groupName, roleName); - String updatedJsonStr = JsonUtils.listToJson(auditFilters); - XXService xService = daoMgr.getXXService().getById(xConfigMap.getServiceId()); - RangerService rangerService = svcService.getPopulatedViewObject(xService); - Map configs = rangerService.getConfigs(); + String updatedJsonStr = JsonUtils.listToJson(auditFilters); + XXService xService = daoMgr.getXXService().getById(xConfigMap.getServiceId()); + RangerService rangerService = svcService.getPopulatedViewObject(xService); + Map configs = rangerService.getConfigs(); if (configs.containsKey(ServiceDBStore.RANGER_PLUGIN_AUDIT_FILTERS)) { - updatedJsonStr = StringUtils.isBlank(updatedJsonStr) ? "" : updatedJsonStr.replaceAll("\"", "'"); + updatedJsonStr = StringUtils.isBlank(updatedJsonStr) ? "" + : updatedJsonStr.replaceAll("\"", "'"); configs.put(ServiceDBStore.RANGER_PLUGIN_AUDIT_FILTERS, updatedJsonStr); try { - LOG.info("==>ServiceDBStore.updateServiceAuditConfig updating audit-filter of service : {} as part of delete request for : {}", rangerService.getName(), searchUsrGrpRoleName); + LOG.info( + "==>ServiceDBStore.updateServiceAuditConfig updating audit-filter of service : {} as part of delete request for : {}", + rangerService.getName(), searchUsrGrpRoleName); updateService(rangerService, null); } catch (Throwable excp) { @@ -2836,32 +3073,38 @@ public void updateServiceAuditConfig(String searchUsrGrpRoleName, REMOVE_REF_TYP } } } else { - LOG.debug("ServiceDBStore.updateServiceAuditConfig no service audit filter Config map found for : {}", searchUsrGrpRoleName); + LOG.debug("ServiceDBStore.updateServiceAuditConfig no service audit filter Config map found for : {}", + searchUsrGrpRoleName); } - LOG.debug("<=== ServiceDBStore.updateServiceAuditConfig( searchUsrGrpRoleName : {} removeRefType : {})", searchUsrGrpRoleName, removeRefType); + LOG.debug("<=== ServiceDBStore.updateServiceAuditConfig( searchUsrGrpRoleName : {} removeRefType : {})", + searchUsrGrpRoleName, removeRefType); } void createTransactionLog(RangerPolicy policy, int operationImportContext, int operationContext) { StackTraceElement[] trace = Thread.currentThread().getStackTrace(); - if (trace.length > 3 && (StringUtils.contains(trace[4].getMethodName(), "import") || StringUtils.contains(trace[5].getMethodName(), "import"))) { + if (trace.length > 3 && (StringUtils.contains(trace[4].getMethodName(), "import") + || StringUtils.contains(trace[5].getMethodName(), "import"))) { policyService.createTransactionLog(policy, null, operationImportContext); } else { policyService.createTransactionLog(policy, null, operationContext); } } - List applyResourceFilter(RangerServiceDef serviceDef, List policies, Map filterResources, SearchFilter filter, RangerPolicyResourceMatcher.MatchScope scope) { - LOG.debug("==> ServiceDBStore.applyResourceFilter(policies-size={}, filterResources={}, {})", policies.size(), filterResources, scope); + List applyResourceFilter(RangerServiceDef serviceDef, List policies, + Map filterResources, SearchFilter filter, RangerPolicyResourceMatcher.MatchScope scope) { + LOG.debug("==> ServiceDBStore.applyResourceFilter(policies-size={}, filterResources={}, {})", policies.size(), + filterResources, scope); - List ret = new ArrayList<>(); + List ret = new ArrayList<>(); List matchers = getMatchers(serviceDef, filterResources, filter); if (CollectionUtils.isNotEmpty(matchers)) { for (RangerPolicy policy : policies) { for (RangerPolicyResourceMatcher matcher : matchers) { - LOG.debug("Trying to match for policy:[{}] using RangerDefaultPolicyResourceMatcher:[{}]", policy, matcher); + LOG.debug("Trying to match for policy:[{}] using RangerDefaultPolicyResourceMatcher:[{}]", policy, + matcher); if (matcher.isMatch(policy, scope, null)) { LOG.debug("matched policy:[{}]", policy); @@ -2873,28 +3116,32 @@ List applyResourceFilter(RangerServiceDef serviceDef, List getMatchers(RangerServiceDef serviceDef, Map filterResources, SearchFilter filter) { + List getMatchers(RangerServiceDef serviceDef, Map filterResources, + SearchFilter filter) { LOG.debug("==> ServiceDBStore.getMatchers(filterResources={})", filterResources); - List ret = new ArrayList<>(); - RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef); - String policyTypeStr = filter.getParam(SearchFilter.POLICY_TYPE); - int[] policyTypes = RangerPolicy.POLICY_TYPES; + List ret = new ArrayList<>(); + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef); + String policyTypeStr = filter.getParam(SearchFilter.POLICY_TYPE); + int[] policyTypes = RangerPolicy.POLICY_TYPES; if (StringUtils.isNotBlank(policyTypeStr)) { - policyTypes = new int[1]; + policyTypes = new int[1]; policyTypes[0] = Integer.parseInt(policyTypeStr); } for (Integer policyType : policyTypes) { - Set> validResourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType, filterResources.keySet()); + Set> validResourceHierarchies = serviceDefHelper.getResourceHierarchies(policyType, + filterResources.keySet()); - LOG.debug("Found {} valid resource hierarchies for key-set {}", validResourceHierarchies.size(), filterResources.keySet()); + LOG.debug("Found {} valid resource hierarchies for key-set {}", validResourceHierarchies.size(), + filterResources.keySet()); List> resourceHierarchies = new ArrayList<>(validResourceHierarchies); @@ -2904,7 +3151,8 @@ List getMatchers(RangerServiceDef serviceDef, Map policyResources = new HashMap<>(); for (RangerResourceDef resourceDef : validResourceHierarchy) { - policyResources.put(resourceDef.getName(), new RangerPolicyResource(filterResources.get(resourceDef.getName()), false, resourceDef.getRecursiveSupported())); + policyResources.put(resourceDef.getName(), new RangerPolicyResource( + filterResources.get(resourceDef.getName()), false, resourceDef.getRecursiveSupported())); } RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); @@ -2924,34 +3172,44 @@ List getMatchers(RangerServiceDef serviceDef, Map ServiceDBStore.getServicePoliciesWithDeltas(serviceType={}, serviceId={}, tagServiceId={}, lastKnownVersion={})", serviceDef.getName(), service.getId(), tagService != null ? tagService.getId() : null, lastKnownVersion); + // if lastKnownVersion != -1L : try and get deltas. Get delta for serviceName + // first. Find id of the delta + // returned first in the list. and then find all ids greater than that for + // corresponding tag service. + LOG.debug( + "==> ServiceDBStore.getServicePoliciesWithDeltas(serviceType={}, serviceId={}, tagServiceId={}, lastKnownVersion={})", + serviceDef.getName(), service.getId(), tagService != null ? tagService.getId() : null, + lastKnownVersion); if (lastKnownVersion != -1L) { - List tagPolicyDeltas = null; - Long retrievedPolicyVersion = null; - Long retrievedTagPolicyVersion = null; - String componentServiceType = serviceDef.getName(); + List tagPolicyDeltas = null; + Long retrievedPolicyVersion = null; + Long retrievedTagPolicyVersion = null; + String componentServiceType = serviceDef.getName(); - List resourcePolicyDeltas = daoMgr.getXXPolicyChangeLog().findLaterThan(lastKnownVersion, maxNeededVersion, service.getId()); + List resourcePolicyDeltas = daoMgr.getXXPolicyChangeLog().findLaterThan(lastKnownVersion, + maxNeededVersion, service.getId()); if (CollectionUtils.isNotEmpty(resourcePolicyDeltas)) { boolean isValid = RangerPolicyDeltaUtil.isValidDeltas(resourcePolicyDeltas, componentServiceType); if (isValid) { - retrievedPolicyVersion = resourcePolicyDeltas.get(resourcePolicyDeltas.size() - 1).getPoliciesVersion(); + retrievedPolicyVersion = resourcePolicyDeltas.get(resourcePolicyDeltas.size() - 1) + .getPoliciesVersion(); } else { - LOG.warn("Resource policy-Deltas :[{}] from version :[{}] are not valid", resourcePolicyDeltas, lastKnownVersion); + LOG.warn("Resource policy-Deltas :[{}] from version :[{}] are not valid", resourcePolicyDeltas, + lastKnownVersion); } if (isValid && tagService != null) { Long id = resourcePolicyDeltas.get(0).getId(); - tagPolicyDeltas = daoMgr.getXXPolicyChangeLog().findGreaterThan(id, maxNeededVersion, tagService.getId()); + tagPolicyDeltas = daoMgr.getXXPolicyChangeLog().findGreaterThan(id, maxNeededVersion, + tagService.getId()); if (CollectionUtils.isNotEmpty(tagPolicyDeltas)) { String tagServiceType = tagServiceDef.getName(); @@ -2959,17 +3217,23 @@ ServicePolicies getServicePoliciesWithDeltas(RangerServiceDef serviceDef, XXServ isValid = RangerPolicyDeltaUtil.isValidDeltas(tagPolicyDeltas, tagServiceType); if (isValid) { - retrievedTagPolicyVersion = tagPolicyDeltas.get(tagPolicyDeltas.size() - 1).getPoliciesVersion(); + retrievedTagPolicyVersion = tagPolicyDeltas.get(tagPolicyDeltas.size() - 1) + .getPoliciesVersion(); } else { - LOG.warn("Tag policy-Deltas :[{}] for service-version :[{}] and delta-id :[{}] are not valid", tagPolicyDeltas, lastKnownVersion, id); + LOG.warn( + "Tag policy-Deltas :[{}] for service-version :[{}] and delta-id :[{}] are not valid", + tagPolicyDeltas, lastKnownVersion, id); } } } if (isValid) { if (CollectionUtils.isNotEmpty(tagPolicyDeltas)) { - // To ensure that resource-policy-deltas with service-type of 'tag' are ignored after validation - resourcePolicyDeltas.removeIf(rangerPolicyDelta -> StringUtils.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME, rangerPolicyDelta.getServiceType())); + // To ensure that resource-policy-deltas with service-type of 'tag' are ignored + // after validation + resourcePolicyDeltas.removeIf(rangerPolicyDelta -> StringUtils.equals( + EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME, + rangerPolicyDelta.getServiceType())); resourcePolicyDeltas.addAll(tagPolicyDeltas); } @@ -2998,15 +3262,20 @@ ServicePolicies getServicePoliciesWithDeltas(RangerServiceDef serviceDef, XXServ ret.setTagPolicies(tagPolicies); } } else { - LOG.warn("Deltas :[{}] from version :[{}] after compressing are null!", resourcePolicyDeltas, lastKnownVersion); + LOG.warn("Deltas :[{}] from version :[{}] after compressing are null!", resourcePolicyDeltas, + lastKnownVersion); } } } else { - LOG.warn("No policy-deltas found for serviceId={}, tagServiceId={}, lastKnownVersion={})", service.getId(), tagService != null ? tagService.getId() : null, lastKnownVersion); + LOG.warn("No policy-deltas found for serviceId={}, tagServiceId={}, lastKnownVersion={})", + service.getId(), tagService != null ? tagService.getId() : null, lastKnownVersion); } } - LOG.debug("<== ServiceDBStore.getServicePoliciesWithDeltas(serviceType={}, serviceId={}, tagServiceId={}, lastKnownVersion={}) : deltasSize={}", serviceDef.getName(), service.getId(), tagService != null ? tagService.getId() : null, lastKnownVersion, ret != null && CollectionUtils.isNotEmpty(ret.getPolicyDeltas()) ? ret.getPolicyDeltas().size() : 0); + LOG.debug( + "<== ServiceDBStore.getServicePoliciesWithDeltas(serviceType={}, serviceId={}, tagServiceId={}, lastKnownVersion={}) : deltasSize={}", + serviceDef.getName(), service.getId(), tagService != null ? tagService.getId() : null, lastKnownVersion, + ret != null && CollectionUtils.isNotEmpty(ret.getPolicyDeltas()) ? ret.getPolicyDeltas().size() : 0); return ret; } @@ -3023,13 +3292,15 @@ void createDefaultPolicies(RangerService createdService) throws Exception { List populateDefaultPolicies(RangerService service) throws Exception { List ret = null; - RangerBaseService svc = serviceMgr.getRangerServiceByService(service, this); + RangerBaseService svc = serviceMgr.getRangerServiceByService(service, this); if (svc != null) { List serviceCheckUsers = getServiceCheckUsers(service); - List users = new ArrayList<>(); + List users = new ArrayList<>(); - /*Need to create ambari service check user before initiating policy creation. */ + /* + * Need to create ambari service check user before initiating policy creation. + */ if (serviceCheckUsers != null) { for (String userName : serviceCheckUsers) { if (!StringUtils.isEmpty(userName)) { @@ -3054,8 +3325,10 @@ List populateDefaultPolicies(RangerService service) throws Excepti createDefaultPolicyUsersAndGroups(defaultPolicies); for (RangerPolicy defaultPolicy : defaultPolicies) { - if (CollectionUtils.isNotEmpty(users) && StringUtils.equalsIgnoreCase(defaultPolicy.getService(), service.getName())) { - RangerPolicyItem defaultAllowPolicyItem = CollectionUtils.isNotEmpty(defaultPolicy.getPolicyItems()) ? defaultPolicy.getPolicyItems().get(0) : null; + if (CollectionUtils.isNotEmpty(users) + && StringUtils.equalsIgnoreCase(defaultPolicy.getService(), service.getName())) { + RangerPolicyItem defaultAllowPolicyItem = CollectionUtils.isNotEmpty( + defaultPolicy.getPolicyItems()) ? defaultPolicy.getPolicyItems().get(0) : null; if (defaultAllowPolicyItem == null) { LOG.error("There is no allow-policy-item in the default-policy:[{}]", defaultPolicy); @@ -3084,7 +3357,8 @@ && validatePolicyItems(defaultPolicy.getDataMaskPolicyItems()) ret.add(defaultPolicy); } else { - LOG.warn("Default policy won't be created,since policyItems not valid-either users/groups not present or access not present in policy."); + LOG.warn( + "Default policy won't be created,since policyItems not valid-either users/groups not present or access not present in policy."); } } } @@ -3094,7 +3368,7 @@ && validatePolicyItems(defaultPolicy.getDataMaskPolicyItems()) } void createDefaultPolicyUsersAndGroups(List defaultPolicies) { - Set defaultPolicyUsers = new HashSet<>(); + Set defaultPolicyUsers = new HashSet<>(); Set defaultPolicyGroups = new HashSet<>(); for (RangerPolicy defaultPolicy : defaultPolicies) { @@ -3132,15 +3406,17 @@ void createDefaultPolicyUsersAndGroups(List defaultPolicies) { for (String policyUser : defaultPolicyUsers) { LOG.debug("Checking policyUser:[{}] for existence", policyUser); - if (StringUtils.isNotBlank(policyUser) && !StringUtils.equals(policyUser, RangerPolicyEngine.USER_CURRENT) && !StringUtils.equals(policyUser, RangerPolicyEngine.RESOURCE_OWNER)) { + if (StringUtils.isNotBlank(policyUser) && !StringUtils.equals(policyUser, RangerPolicyEngine.USER_CURRENT) + && !StringUtils.equals(policyUser, RangerPolicyEngine.RESOURCE_OWNER)) { String userName = stringUtil.getValidUserName(policyUser); - XXUser xxUser = daoMgr.getXXUser().findByUserName(userName); + XXUser xxUser = daoMgr.getXXUser().findByUserName(userName); if (xxUser == null) { UserSessionBase usb = ContextUtil.getCurrentUserSession(); if (usb != null && !usb.isKeyAdmin() && !usb.isUserAdmin() && !usb.isSpnegoEnabled()) { - throw restErrorUtil.createRESTException("User does not exist with given username: [" + policyUser + "] please use existing user", MessageEnums.OPER_NO_PERMISSION); + throw restErrorUtil.createRESTException("User does not exist with given username: [" + + policyUser + "] please use existing user", MessageEnums.OPER_NO_PERMISSION); } xUserMgr.createServiceConfigUser(userName); @@ -3158,7 +3434,8 @@ void createDefaultPolicyUsersAndGroups(List defaultPolicies) { UserSessionBase usb = ContextUtil.getCurrentUserSession(); if (usb != null && !usb.isKeyAdmin() && !usb.isUserAdmin() && !usb.isSpnegoEnabled()) { - throw restErrorUtil.createRESTException("Group does not exist with given groupname: [" + policyGroup + "] please use existing group", MessageEnums.OPER_NO_PERMISSION); + throw restErrorUtil.createRESTException("Group does not exist with given groupname: [" + + policyGroup + "] please use existing group", MessageEnums.OPER_NO_PERMISSION); } VXGroup vXGroup = new VXGroup(); @@ -3177,8 +3454,8 @@ void createDefaultPolicyUsersAndGroups(List defaultPolicies) { } List getServiceCheckUsers(RangerService createdService) { - List ret = new ArrayList<>(); - String userNames = ""; + List ret = new ArrayList<>(); + String userNames = ""; Map serviceConfig = createdService.getConfigs(); if (serviceConfig.containsKey(SERVICE_CHECK_USER)) { @@ -3210,7 +3487,7 @@ void updatePolicySignature(RangerPolicy policy) { } RangerPolicyResourceSignature policySignature = factory.createPolicyResourceSignature(policy); - String signature = policySignature.getSignature(); + String signature = policySignature.getSignature(); policy.setResourceSignature(signature); @@ -3218,7 +3495,7 @@ void updatePolicySignature(RangerPolicy policy) { } boolean hasServiceConfigForPluginChanged(List dbConfigMaps, Map validConfigs) { - boolean ret = false; + boolean ret = false; Map configs = new HashMap<>(); if (CollectionUtils.isNotEmpty(dbConfigMaps)) { @@ -3252,28 +3529,31 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List resources, List accessTypes, List policyConditions, List contextEnrichers, List enums, RangerDataMaskDef dataMaskDef, RangerRowFilterDef rowFilterDef) { - Long serviceDefId = createdSvcDef.getId(); - List xxConfigs = daoMgr.getXXServiceConfigDef().findByServiceDefId(serviceDefId); - List xxResources = daoMgr.getXXResourceDef().findByServiceDefId(serviceDefId); - List xxAccessTypes = daoMgr.getXXAccessTypeDef().findByServiceDefId(serviceDefId); - List xxPolicyConditions = daoMgr.getXXPolicyConditionDef().findByServiceDefId(serviceDefId); - List xxContextEnrichers = daoMgr.getXXContextEnricherDef().findByServiceDefId(serviceDefId); - List xxEnums = daoMgr.getXXEnumDef().findByServiceDefId(serviceDefId); - XXServiceConfigDefDao xxServiceConfigDao = daoMgr.getXXServiceConfigDef(); + Long serviceDefId = createdSvcDef.getId(); + List xxConfigs = daoMgr.getXXServiceConfigDef().findByServiceDefId(serviceDefId); + List xxResources = daoMgr.getXXResourceDef().findByServiceDefId(serviceDefId); + List xxAccessTypes = daoMgr.getXXAccessTypeDef().findByServiceDefId(serviceDefId); + List xxPolicyConditions = daoMgr.getXXPolicyConditionDef() + .findByServiceDefId(serviceDefId); + List xxContextEnrichers = daoMgr.getXXContextEnricherDef() + .findByServiceDefId(serviceDefId); + List xxEnums = daoMgr.getXXEnumDef().findByServiceDefId(serviceDefId); + XXServiceConfigDefDao xxServiceConfigDao = daoMgr.getXXServiceConfigDef(); for (int i = 0; i < configs.size(); i++) { RangerServiceConfigDef config = configs.get(i); - boolean found = false; + boolean found = false; for (XXServiceConfigDef xConfig : xxConfigs) { if (config.getItemId() != null && config.getItemId().equals(xConfig.getItemId())) { - found = true; - xConfig = serviceDefService.populateRangerServiceConfigDefToXX(config, xConfig, createdSvcDef, RangerServiceDefService.OPERATION_UPDATE_CONTEXT); + found = true; + xConfig = serviceDefService.populateRangerServiceConfigDefToXX(config, xConfig, createdSvcDef, + RangerServiceDefService.OPERATION_UPDATE_CONTEXT); xConfig.setOrder(i); xConfig = xxServiceConfigDao.update(xConfig); - config = serviceDefService.populateXXToRangerServiceConfigDef(xConfig); + config = serviceDefService.populateXXToRangerServiceConfigDef(xConfig); break; } } @@ -3281,7 +3561,8 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List xxPolicyRefResource = daoMgr.getXXPolicyRefResource().findByResourceDefID(xRes.getId()); + List xxPolicyRefResource = daoMgr.getXXPolicyRefResource() + .findByResourceDefID(xRes.getId()); if (!stringUtil.isEmpty(xxPolicyRefResource)) { - throw restErrorUtil.createRESTException("Policy/Policies are referring to this resource: " + xRes.getName() + ". Please remove such references from policy before updating service-def.", MessageEnums.DATA_NOT_UPDATABLE); + throw restErrorUtil.createRESTException( + "Policy/Policies are referring to this resource: " + xRes.getName() + + ". Please remove such references from policy before updating service-def.", + MessageEnums.DATA_NOT_UPDATABLE); } deleteXXResourceDef(xRes); @@ -3361,20 +3648,21 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List impliedGrants = access.getImpliedGrants(); - XXAccessTypeDefGrantsDao xxATDGrantDao = daoMgr.getXXAccessTypeDefGrants(); - List xxImpliedGrants = xxATDGrantDao.findImpliedGrantsByATDId(xAccess.getId()); + Collection impliedGrants = access.getImpliedGrants(); + XXAccessTypeDefGrantsDao xxATDGrantDao = daoMgr.getXXAccessTypeDefGrants(); + List xxImpliedGrants = xxATDGrantDao.findImpliedGrantsByATDId(xAccess.getId()); for (String impliedGrant : impliedGrants) { boolean foundGrant = false; @@ -3407,7 +3695,8 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List impliedGrants = access.getImpliedGrants(); + Collection impliedGrants = access.getImpliedGrants(); XXAccessTypeDefGrantsDao xxATDGrantDao = daoMgr.getXXAccessTypeDefGrants(); for (String impliedGrant : impliedGrants) { @@ -3454,10 +3744,14 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List policyRefAccessTypeList = daoMgr.getXXPolicyRefAccessType().findByAccessTypeDefId(xAccess.getId()); + List policyRefAccessTypeList = daoMgr.getXXPolicyRefAccessType() + .findByAccessTypeDefId(xAccess.getId()); if (!stringUtil.isEmpty(policyRefAccessTypeList)) { - throw restErrorUtil.createRESTException("Policy/Policies are referring to this access-type: " + xAccess.getName() + ". Please remove such references from policy before updating service-def.", MessageEnums.DATA_NOT_UPDATABLE); + throw restErrorUtil.createRESTException( + "Policy/Policies are referring to this access-type: " + xAccess.getName() + + ". Please remove such references from policy before updating service-def.", + MessageEnums.DATA_NOT_UPDATABLE); } deleteXXAccessTypeDef(xAccess); @@ -3468,17 +3762,18 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List xxPolicyRefConditions = daoMgr.getXXPolicyRefCondition().findByConditionDefId(xCondition.getId()); + List xxPolicyRefConditions = daoMgr.getXXPolicyRefCondition() + .findByConditionDefId(xCondition.getId()); if (!stringUtil.isEmpty(xxPolicyRefConditions)) { - throw restErrorUtil.createRESTException("Policy/Policies are referring to this policy-condition: " + xCondition.getName() + ". Please remove such references from policy before updating service-def.", MessageEnums.DATA_NOT_UPDATABLE); + throw restErrorUtil.createRESTException( + "Policy/Policies are referring to this policy-condition: " + xCondition.getName() + + ". Please remove such references from policy before updating service-def.", + MessageEnums.DATA_NOT_UPDATABLE); } for (XXPolicyRefCondition xxPolicyRefCondition : xxPolicyRefConditions) { @@ -3525,17 +3825,18 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List xxEnumEleDefs = xEnumEleDao.findByEnumDefId(xEnumDef.getId()); - List enumEleDefs = enumDef.getElements(); + XXEnumElementDefDao xEnumEleDao = daoMgr.getXXEnumElementDef(); + List xxEnumEleDefs = xEnumEleDao.findByEnumDefId(xEnumDef.getId()); + List enumEleDefs = enumDef.getElements(); for (int i = 0; i < enumEleDefs.size(); i++) { - RangerEnumElementDef eleDef = enumEleDefs.get(i); - boolean foundEle = false; + RangerEnumElementDef eleDef = enumEleDefs.get(i); + boolean foundEle = false; for (XXEnumElementDef xEleDef : xxEnumEleDefs) { if (eleDef.getItemId() != null && eleDef.getItemId().equals(xEleDef.getItemId())) { foundEle = true; - xEleDef = serviceDefService.populateRangerEnumElementDefToXX(eleDef, xEleDef, xEnumDef, RangerServiceDefService.OPERATION_UPDATE_CONTEXT); + xEleDef = serviceDefService.populateRangerEnumElementDefToXX(eleDef, xEleDef, xEnumDef, + RangerServiceDefService.OPERATION_UPDATE_CONTEXT); xEleDef.setOrder(i); @@ -3602,7 +3906,8 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List elements = enumDef.getElements(); - XXEnumElementDefDao xxEnumEleDefDao = daoMgr.getXXEnumElementDef(); + List elements = enumDef.getElements(); + XXEnumElementDefDao xxEnumEleDefDao = daoMgr.getXXEnumElementDef(); for (RangerEnumElementDef element : elements) { XXEnumElementDef xElement = new XXEnumElementDef(); - xElement = serviceDefService.populateRangerEnumElementDefToXX(element, xElement, xEnum, RangerServiceDefService.OPERATION_CREATE_CONTEXT); + xElement = serviceDefService.populateRangerEnumElementDefToXX(element, xElement, xEnum, + RangerServiceDefService.OPERATION_CREATE_CONTEXT); xxEnumEleDefDao.create(xElement); } @@ -3673,32 +3980,43 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List dataMasks = dataMaskDef == null || dataMaskDef.getMaskTypes() == null ? new ArrayList<>() : dataMaskDef.getMaskTypes(); - List dataMaskAccessTypes = dataMaskDef == null || dataMaskDef.getAccessTypes() == null ? new ArrayList<>() : dataMaskDef.getAccessTypes(); - List dataMaskResources = dataMaskDef == null || dataMaskDef.getResources() == null ? new ArrayList<>() : dataMaskDef.getResources(); - List rowFilterAccessTypes = rowFilterDef == null || rowFilterDef.getAccessTypes() == null ? new ArrayList<>() : rowFilterDef.getAccessTypes(); - List rowFilterResources = rowFilterDef == null || rowFilterDef.getResources() == null ? new ArrayList<>() : rowFilterDef.getResources(); - XXDataMaskTypeDefDao dataMaskTypeDao = daoMgr.getXXDataMaskTypeDef(); - List xxDataMaskTypes = dataMaskTypeDao.findByServiceDefId(serviceDefId); - List xxAccessTypeDefs = xxATDDao.findByServiceDefId(serviceDefId); - List xxResourceDefs = xxResDefDao.findByServiceDefId(serviceDefId); + List dataMasks = dataMaskDef == null || dataMaskDef.getMaskTypes() == null + ? new ArrayList<>() + : dataMaskDef.getMaskTypes(); + List dataMaskAccessTypes = dataMaskDef == null || dataMaskDef.getAccessTypes() == null + ? new ArrayList<>() + : dataMaskDef.getAccessTypes(); + List dataMaskResources = dataMaskDef == null || dataMaskDef.getResources() == null + ? new ArrayList<>() + : dataMaskDef.getResources(); + List rowFilterAccessTypes = rowFilterDef == null || rowFilterDef.getAccessTypes() == null + ? new ArrayList<>() + : rowFilterDef.getAccessTypes(); + List rowFilterResources = rowFilterDef == null || rowFilterDef.getResources() == null + ? new ArrayList<>() + : rowFilterDef.getResources(); + XXDataMaskTypeDefDao dataMaskTypeDao = daoMgr.getXXDataMaskTypeDef(); + List xxDataMaskTypes = dataMaskTypeDao.findByServiceDefId(serviceDefId); + List xxAccessTypeDefs = xxATDDao.findByServiceDefId(serviceDefId); + List xxResourceDefs = xxResDefDao.findByServiceDefId(serviceDefId); // create or update dataMasks for (int i = 0; i < dataMasks.size(); i++) { RangerDataMaskTypeDef dataMask = dataMasks.get(i); - boolean found = false; + boolean found = false; for (XXDataMaskTypeDef xxDataMask : xxDataMaskTypes) { if (xxDataMask.getItemId() != null && xxDataMask.getItemId().equals(dataMask.getItemId())) { LOG.debug("Updating existing dataMask with itemId={}", dataMask.getItemId()); - found = true; - xxDataMask = serviceDefService.populateRangerDataMaskDefToXX(dataMask, xxDataMask, createdSvcDef, RangerServiceDefService.OPERATION_UPDATE_CONTEXT); + found = true; + xxDataMask = serviceDefService.populateRangerDataMaskDefToXX(dataMask, xxDataMask, createdSvcDef, + RangerServiceDefService.OPERATION_UPDATE_CONTEXT); xxDataMask.setOrder(i); xxDataMask = dataMaskTypeDao.update(xxDataMask); - dataMask = serviceDefService.populateXXToRangerDataMaskTypeDef(xxDataMask); + dataMask = serviceDefService.populateXXToRangerDataMaskTypeDef(xxDataMask); break; } } @@ -3708,7 +4026,8 @@ private void updateChildObjectsOfServiceDef(XXServiceDef createdSvcDef, List policiesToDelete) + throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.cleanupS3BucketPoliciesForService(" + service.getName() + ")"); + } + + try { + Map configs = service.getConfigs(); + S3Client s3 = S3ClientConnectionMgr.getS3client(configs); + + if (s3 == null) { + LOG.warn("S3 client initialization failed for service: " + service.getName()); + return; + } + + String bucketName = configs.get(RangerS3Constants.BUCKET_NAME); + + // Collect all affected buckets from policies + Set affectedBuckets = new HashSet<>(); + for (RangerPolicy policy : policiesToDelete) { + if (policy.getResources() != null) { + for (RangerPolicyResource resource : policy.getResources().values()) { + for (String s3path : resource.getValues()) { + if (s3path.startsWith("*")) { + affectedBuckets.add(bucketName); + } else { + String bucketPart = s3path.split("/", 2)[0]; + affectedBuckets.add(bucketPart); + } + } + } + } + } + + // Delete bucket policies for all affected buckets + // Clean up S3 bucket policies for all affected buckets + for (String bucket : affectedBuckets) { + try { + // When service is deleted, remove all Ranger statements for this service + // but preserve any manually created IAM statements + List emptyRangerStatements = new ArrayList<>(); + List mergedStatements = mergeWithIAMStatements(s3, bucket, emptyRangerStatements, + Collections.emptySet()); + + if (mergedStatements.isEmpty()) { + // No IAM statements to preserve, delete entire policy + deleteBucketPolicy(s3, bucket); + if (LOG.isDebugEnabled()) { + LOG.debug("Deleted entire bucket policy (no IAM statements) for bucket: " + bucket); + } + } else { + // Preserve IAM statements + String finalPolicyJson = mapToS3BucketPolicyObject(mergedStatements); + putBucketPolicy(s3, bucket, finalPolicyJson); + if (LOG.isDebugEnabled()) { + LOG.debug("Updated bucket policy (preserved {} IAM statements) for bucket: " + bucket, + mergedStatements.size()); + } + } + } catch (Exception e) { + LOG.error("Failed to update bucket policy for bucket: " + bucket, e); + // Continue with other buckets even if one fails + } + } + } catch (Exception e) { + LOG.error("Error in cleanupS3BucketPoliciesForService for service: " + service.getName(), e); + throw e; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.cleanupS3BucketPoliciesForService(" + service.getName() + ")"); + } + } + private void updateTabPermissions(String svcType, Map svcConfig) { if (StringUtils.equalsIgnoreCase(svcType, EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { String svcAdminUsers = svcConfig.get(SERVICE_ADMIN_USERS); @@ -3827,16 +4231,21 @@ private void validateUserAndProvideTabTagBasedPolicyPermission(String username) XXPortalUser xxPortalUser = daoMgr.getXXPortalUser().findByLoginId(username); if (xxPortalUser == null) { - throw restErrorUtil.createRESTException("Username : " + username + " does not exist. Please provide valid user as service admin for tag service .", MessageEnums.ERROR_CREATING_OBJECT); + throw restErrorUtil.createRESTException( + "Username : " + username + + " does not exist. Please provide valid user as service admin for tag service .", + MessageEnums.ERROR_CREATING_OBJECT); } else { VXPortalUser vXPortalUser = userMgr.mapXXPortalUserToVXPortalUserForDefaultAccount(xxPortalUser); - if (CollectionUtils.isNotEmpty(vXPortalUser.getUserRoleList()) && vXPortalUser.getUserRoleList().size() == 1) { + if (CollectionUtils.isNotEmpty(vXPortalUser.getUserRoleList()) + && vXPortalUser.getUserRoleList().size() == 1) { for (String userRole : vXPortalUser.getUserRoleList()) { if (userRole.equals(RangerConstants.ROLE_USER)) { HashMap moduleNameId = xUserMgr.getAllModuleNameAndIdMap(); - xUserMgr.createOrUpdateUserPermisson(vXPortalUser, moduleNameId.get(RangerConstants.MODULE_TAG_BASED_POLICIES), true); + xUserMgr.createOrUpdateUserPermisson(vXPortalUser, + moduleNameId.get(RangerConstants.MODULE_TAG_BASED_POLICIES), true); } } } @@ -3850,19 +4259,23 @@ private boolean validatePolicyItems(List policyItems return false; } - if (CollectionUtils.isEmpty(policyItem.getUsers()) && CollectionUtils.isEmpty(policyItem.getGroups()) && CollectionUtils.isEmpty(policyItem.getRoles())) { + if (CollectionUtils.isEmpty(policyItem.getUsers()) && CollectionUtils.isEmpty(policyItem.getGroups()) + && CollectionUtils.isEmpty(policyItem.getRoles())) { return false; } - if (policyItem.getUsers() != null && (policyItem.getUsers().contains(null) || policyItem.getUsers().contains(""))) { + if (policyItem.getUsers() != null + && (policyItem.getUsers().contains(null) || policyItem.getUsers().contains(""))) { return false; } - if (policyItem.getGroups() != null && (policyItem.getGroups().contains(null) || policyItem.getGroups().contains(""))) { + if (policyItem.getGroups() != null + && (policyItem.getGroups().contains(null) || policyItem.getGroups().contains(""))) { return false; } - if (policyItem.getRoles() != null && (policyItem.getRoles().contains(null) || policyItem.getRoles().contains(""))) { + if (policyItem.getRoles() != null + && (policyItem.getRoles().contains(null) || policyItem.getRoles().contains(""))) { return false; } @@ -3881,10 +4294,11 @@ private boolean validatePolicyItems(List policyItems return true; } - private List searchRangerTagPoliciesOnBasisOfServiceName(List allExceptTagPolicies) throws Exception { - List ret = new ArrayList<>(); - Set serviceNames = new HashSet<>(); - Map tagServices = new HashMap<>(); + private List searchRangerTagPoliciesOnBasisOfServiceName(List allExceptTagPolicies) + throws Exception { + List ret = new ArrayList<>(); + Set serviceNames = new HashSet<>(); + Map tagServices = new HashMap<>(); for (RangerPolicy pol : allExceptTagPolicies) { serviceNames.add(pol.getService()); @@ -3904,10 +4318,11 @@ private List searchRangerTagPoliciesOnBasisOfServiceName(List entry : tagServices.entrySet()) { String tagServiceName = entry.getKey(); - Long tagServiceId = entry.getValue(); + Long tagServiceId = entry.getValue(); - ServicePolicies tagServicePolicies = RangerServicePoliciesCache.getInstance().getServicePolicies(tagServiceName, tagServiceId, -1L, true, this); - List policies = tagServicePolicies != null ? tagServicePolicies.getPolicies() : null; + ServicePolicies tagServicePolicies = RangerServicePoliciesCache.getInstance() + .getServicePolicies(tagServiceName, tagServiceId, -1L, true, this); + List policies = tagServicePolicies != null ? tagServicePolicies.getPolicies() : null; if (policies != null) { ret.addAll(policies); @@ -3924,15 +4339,16 @@ private List getServicePolicies(XXService service, SearchFilter fi throw new Exception("service does not exist"); } - List ret; - ServicePolicies servicePolicies = RangerServicePoliciesCache.getInstance().getServicePolicies(service.getName(), service.getId(), -1L, true, this); - final List policies = servicePolicies != null ? servicePolicies.getPolicies() : null; + List ret; + ServicePolicies servicePolicies = RangerServicePoliciesCache.getInstance().getServicePolicies(service.getName(), + service.getId(), -1L, true, this); + final List policies = servicePolicies != null ? servicePolicies.getPolicies() : null; if (policies != null && filter != null && MapUtils.isNotEmpty(filter.getParams())) { - Map filterResources = filter.getParamsWithPrefix(SearchFilter.RESOURCE_PREFIX, true); - String resourceMatchScope = filter.getParam(SearchFilter.RESOURCE_MATCH_SCOPE); - boolean useLegacyResourceSearch = true; - SearchFilter searchFilter = new SearchFilter(filter); + Map filterResources = filter.getParamsWithPrefix(SearchFilter.RESOURCE_PREFIX, true); + String resourceMatchScope = filter.getParam(SearchFilter.RESOURCE_MATCH_SCOPE); + boolean useLegacyResourceSearch = true; + SearchFilter searchFilter = new SearchFilter(filter); if (MapUtils.isNotEmpty(filterResources) && resourceMatchScope != null) { useLegacyResourceSearch = false; @@ -3970,7 +4386,8 @@ private List getServicePolicies(XXService service, SearchFilter fi break; } case ANCESTOR: { - Map updatedFilterResources = RangerServiceDefHelper.getFilterResourcesForAncestorPolicyFiltering(serviceDef, filterResources); + Map updatedFilterResources = RangerServiceDefHelper + .getFilterResourcesForAncestorPolicyFiltering(serviceDef, filterResources); if (MapUtils.isNotEmpty(updatedFilterResources)) { filterResources.putAll(updatedFilterResources); @@ -3998,18 +4415,20 @@ private List getServicePoliciesFromDb(XXService service) { LOG.debug("==> ServiceDBStore.getServicePoliciesFromDb({})", service.getName()); RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager); - List ret = policyRetriever.getServicePolicies(service); + List ret = policyRetriever.getServicePolicies(service); - LOG.debug("<== ServiceDBStore.getServicePoliciesFromDb({}): count={}", service.getName(), (ret == null) ? 0 : ret.size()); + LOG.debug("<== ServiceDBStore.getServicePoliciesFromDb({}): count={}", service.getName(), + (ret == null) ? 0 : ret.size()); return ret; } - private ServicePolicies getServicePolicies(String serviceName, Long lastKnownVersion, boolean getOnlyDeltas, boolean isDeltaEnabled, Long maxNeededVersion) throws Exception { + private ServicePolicies getServicePolicies(String serviceName, Long lastKnownVersion, boolean getOnlyDeltas, + boolean isDeltaEnabled, Long maxNeededVersion) throws Exception { LOG.debug("==> ServiceDBStore.getServicePolicies({}, {})", serviceName, lastKnownVersion); - ServicePolicies ret = null; - XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName); + ServicePolicies ret = null; + XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName); if (serviceDbObj == null) { throw new Exception("service does not exist. name=" + serviceName); @@ -4027,10 +4446,10 @@ private ServicePolicies getServicePolicies(String serviceName, Long lastKnownVer throw new Exception("service-def does not exist. id=" + serviceDbObj.getType()); } - String serviceType = serviceDef.getName(); - String auditMode = getAuditMode(serviceType, serviceName); - XXService tagServiceDbObj = null; - RangerServiceDef tagServiceDef = null; + String serviceType = serviceDef.getName(); + String auditMode = getAuditMode(serviceType, serviceName); + XXService tagServiceDbObj = null; + RangerServiceDef tagServiceDef = null; XXServiceVersionInfo tagServiceVersionInfoDbObj = null; if (serviceDbObj.getTagService() != null) { @@ -4054,16 +4473,19 @@ private ServicePolicies getServicePolicies(String serviceName, Long lastKnownVer } if (isDeltaEnabled) { - ret = getServicePoliciesWithDeltas(serviceDef, serviceDbObj, tagServiceDef, tagServiceDbObj, lastKnownVersion, maxNeededVersion); + ret = getServicePoliciesWithDeltas(serviceDef, serviceDbObj, tagServiceDef, tagServiceDbObj, + lastKnownVersion, maxNeededVersion); } if (ret != null) { - ret.setPolicyUpdateTime(serviceVersionInfoDbObj == null ? null : serviceVersionInfoDbObj.getPolicyUpdateTime()); + ret.setPolicyUpdateTime( + serviceVersionInfoDbObj == null ? null : serviceVersionInfoDbObj.getPolicyUpdateTime()); ret.setAuditMode(auditMode); ret.setServiceConfig(getServiceConfigForPlugin(serviceDbObj.getId())); if (ret.getTagPolicies() != null) { - ret.getTagPolicies().setPolicyUpdateTime(tagServiceVersionInfoDbObj == null ? null : tagServiceVersionInfoDbObj.getPolicyUpdateTime()); + ret.getTagPolicies().setPolicyUpdateTime( + tagServiceVersionInfoDbObj == null ? null : tagServiceVersionInfoDbObj.getPolicyUpdateTime()); ret.getTagPolicies().setAuditMode(auditMode); } } else if (!getOnlyDeltas) { @@ -4075,8 +4497,10 @@ private ServicePolicies getServicePolicies(String serviceName, Long lastKnownVer tagPolicies.setServiceId(tagServiceDbObj.getId()); tagPolicies.setServiceName(tagServiceDbObj.getName()); tagPolicies.setServiceConfig(getServiceConfigForPlugin(tagServiceDbObj.getId())); - tagPolicies.setPolicyVersion(tagServiceVersionInfoDbObj == null ? null : tagServiceVersionInfoDbObj.getPolicyVersion()); - tagPolicies.setPolicyUpdateTime(tagServiceVersionInfoDbObj == null ? null : tagServiceVersionInfoDbObj.getPolicyUpdateTime()); + tagPolicies.setPolicyVersion( + tagServiceVersionInfoDbObj == null ? null : tagServiceVersionInfoDbObj.getPolicyVersion()); + tagPolicies.setPolicyUpdateTime( + tagServiceVersionInfoDbObj == null ? null : tagServiceVersionInfoDbObj.getPolicyUpdateTime()); tagPolicies.setPolicies(getServicePoliciesFromDb(tagServiceDbObj)); tagPolicies.setServiceDef(tagServiceDef); tagPolicies.setAuditMode(auditMode); @@ -4090,26 +4514,30 @@ private ServicePolicies getServicePolicies(String serviceName, Long lastKnownVer ret.setServiceName(serviceDbObj.getName()); ret.setServiceConfig(getServiceConfigForPlugin(ret.getServiceId())); ret.setPolicyVersion(serviceVersionInfoDbObj == null ? null : serviceVersionInfoDbObj.getPolicyVersion()); - ret.setPolicyUpdateTime(serviceVersionInfoDbObj == null ? null : serviceVersionInfoDbObj.getPolicyUpdateTime()); + ret.setPolicyUpdateTime( + serviceVersionInfoDbObj == null ? null : serviceVersionInfoDbObj.getPolicyUpdateTime()); ret.setPolicies(policies); ret.setServiceDef(serviceDef); ret.setAuditMode(auditMode); ret.setTagPolicies(tagPolicies); } - LOG.debug("ServiceDBStore.getServicePolicies({}, {}): ret = {}", serviceName, lastKnownVersion, ret == null ? null : ret.getServiceConfig()); + LOG.debug("ServiceDBStore.getServicePolicies({}, {}): ret = {}", serviceName, lastKnownVersion, + ret == null ? null : ret.getServiceConfig()); - LOG.debug("<== ServiceDBStore.getServicePolicies({}, {}): count={}, delta-count={}", serviceName, lastKnownVersion, (ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size(), (ret == null || ret.getPolicyDeltas() == null) ? 0 : ret.getPolicyDeltas().size()); + LOG.debug("<== ServiceDBStore.getServicePolicies({}, {}): count={}, delta-count={}", serviceName, + lastKnownVersion, (ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size(), + (ret == null || ret.getPolicyDeltas() == null) ? 0 : ret.getPolicyDeltas().size()); return ret; } private static List compressDeltas(List deltas) { - List ret = new ArrayList<>(); + List ret = new ArrayList<>(); final Map> policyDeltaMap = new HashMap<>(); for (RangerPolicyDelta delta : deltas) { - Long policyId = delta.getPolicyId(); + Long policyId = delta.getPolicyId(); List oldPolicyDeltas = policyDeltaMap.computeIfAbsent(policyId, k -> new ArrayList<>()); oldPolicyDeltas.add(delta); @@ -4122,7 +4550,7 @@ private static List compressDeltas(List de ret.addAll(policyDeltas); } else { // Will always be greater than 1 List policyDeltasForPolicy = new ArrayList<>(); - RangerPolicyDelta first = policyDeltas.get(0); + RangerPolicyDelta first = policyDeltas.get(0); policyDeltasForPolicy.add(first); @@ -4161,7 +4589,9 @@ private static List compressDeltas(List de policyDeltasForPolicy.clear(); index++; } else { - LOG.error("CHANGE_TYPE_POLICY_DELETE should be the last policyDelta, found:[{}]", policyDeltas.get(index + 1)); + LOG.error( + "CHANGE_TYPE_POLICY_DELETE should be the last policyDelta, found:[{}]", + policyDeltas.get(index + 1)); policyDeltasForPolicy = null; } @@ -4180,7 +4610,9 @@ private static List compressDeltas(List de switch (policyDelta.getChangeType()) { case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE: - LOG.error("Should not get here! policy is created after it is updated!! policy-delta:[{}]", policyDelta); + LOG.error( + "Should not get here! policy is created after it is updated!! policy-delta:[{}]", + policyDelta); policyDeltasForPolicy = null; break; @@ -4206,7 +4638,9 @@ private static List compressDeltas(List de policyDeltasForPolicy.add(policyDeltas.get(index)); index++; } else { - LOG.error("CHANGE_TYPE_POLICY_DELETE should be the last policyDelta, found:[{}]", policyDeltas.get(index + 1)); + LOG.error( + "CHANGE_TYPE_POLICY_DELETE should be the last policyDelta, found:[{}]", + policyDeltas.get(index + 1)); policyDeltasForPolicy = null; } @@ -4221,7 +4655,8 @@ private static List compressDeltas(List de } break; case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE: - LOG.error("CHANGE_TYPE_POLICY_DELETE should be the last policyDelta, found:[{}]", policyDeltas.get(index)); + LOG.error("CHANGE_TYPE_POLICY_DELETE should be the last policyDelta, found:[{}]", + policyDeltas.get(index)); policyDeltasForPolicy = null; break; @@ -4231,7 +4666,8 @@ private static List compressDeltas(List de } if (policyDeltasForPolicy != null) { - LOG.debug("Processed deltas for policy:[{}], compressed-deltas:[{}]", entry.getKey(), policyDeltasForPolicy); + LOG.debug("Processed deltas for policy:[{}], compressed-deltas:[{}]", entry.getKey(), + policyDeltasForPolicy); ret.addAll(policyDeltasForPolicy); } else { @@ -4279,13 +4715,15 @@ private Map validateRequiredConfigParams(RangerService service, return null; } - List svcConfDefList = daoMgr.getXXServiceConfigDef().findByServiceDefName(service.getType()); + List svcConfDefList = daoMgr.getXXServiceConfigDef() + .findByServiceDefName(service.getType()); for (XXServiceConfigDef svcConfDef : svcConfDefList) { String confField = configs.get(svcConfDef.getName()); if (svcConfDef.getIsMandatory() && stringUtil.isEmpty(confField)) { - throw restErrorUtil.createRESTException("Please provide value of mandatory: " + svcConfDef.getName(), MessageEnums.INVALID_INPUT_DATA); + throw restErrorUtil.createRESTException("Please provide value of mandatory: " + svcConfDef.getName(), + MessageEnums.INVALID_INPUT_DATA); } if (StringUtils.equals(svcConfDef.getName(), RANGER_PLUGIN_AUDIT_FILTERS)) { @@ -4293,7 +4731,8 @@ private Map validateRequiredConfigParams(RangerService service, configs.put(RANGER_PLUGIN_AUDIT_FILTERS, svcConfDef.getDefaultvalue()); } - if (!stringUtil.isEmpty(configs.get(RANGER_PLUGIN_AUDIT_FILTERS)) && JsonUtils.jsonToAuditFilterList(configs.get(RANGER_PLUGIN_AUDIT_FILTERS)) == null) { + if (!stringUtil.isEmpty(configs.get(RANGER_PLUGIN_AUDIT_FILTERS)) + && JsonUtils.jsonToAuditFilterList(configs.get(RANGER_PLUGIN_AUDIT_FILTERS)) == null) { throw restErrorUtil.createRESTException("Invalid value for " + svcConfDef.getName()); } } @@ -4310,16 +4749,18 @@ private Map validateRequiredConfigParams(RangerService service, return validConfigs; } - private void handlePolicyUpdate(RangerService service, Integer policyDeltaType, RangerPolicy policy, boolean updateServiceInfoRoleVersion) { + private void handlePolicyUpdate(RangerService service, Integer policyDeltaType, RangerPolicy policy, + boolean updateServiceInfoRoleVersion) { updatePolicyVersion(service, policyDeltaType, policy, updateServiceInfoRoleVersion); } - private void updatePolicyVersion(RangerService service, Integer policyDeltaType, RangerPolicy policy, boolean updateServiceInfoRoleVersion) { + private void updatePolicyVersion(RangerService service, Integer policyDeltaType, RangerPolicy policy, + boolean updateServiceInfoRoleVersion) { if (service == null || service.getId() == null) { return; } - XXServiceDao serviceDao = daoMgr.getXXService(); + XXServiceDao serviceDao = daoMgr.getXXService(); final XXService serviceDbObj = serviceDao.getById(service.getId()); if (serviceDbObj == null) { @@ -4329,7 +4770,7 @@ private void updatePolicyVersion(RangerService service, Integer policyDeltaType, } final RangerDaoManager daoManager = daoMgr; - final Long serviceId = serviceDbObj.getId(); + final Long serviceId = serviceDbObj.getId(); // if this is a tag/gds service, update all services that refer to this service // so that next policy-download from plugins will get updated tag/gds policies @@ -4339,12 +4780,16 @@ private void updatePolicyVersion(RangerService service, Integer policyDeltaType, List referringServiceIds = serviceDao.findIdsByTagServiceId(serviceId); for (Long referringServiceId : referringServiceIds) { - Runnable policyVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, VERSION_TYPE.POLICY_VERSION, policy != null ? policy.getZoneName() : null, policyDeltaType, policy); + Runnable policyVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, + VERSION_TYPE.POLICY_VERSION, policy != null ? policy.getZoneName() : null, policyDeltaType, + policy); transactionSynchronizationAdapter.executeOnTransactionCommit(policyVersionUpdater); if (updateServiceInfoRoleVersion) { - Runnable roleVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, VERSION_TYPE.ROLE_VERSION, policy != null ? policy.getZoneName() : null, policyDeltaType, policy); + Runnable roleVersionUpdater = new ServiceVersionUpdater(daoManager, referringServiceId, + VERSION_TYPE.ROLE_VERSION, policy != null ? policy.getZoneName() : null, policyDeltaType, + policy); transactionSynchronizationAdapter.executeOnTransactionCommit(roleVersionUpdater); } @@ -4353,19 +4798,22 @@ private void updatePolicyVersion(RangerService service, Integer policyDeltaType, final VERSION_TYPE versionType = VERSION_TYPE.POLICY_VERSION; - Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType, policy != null ? policy.getZoneName() : null, policyDeltaType, policy); + Runnable serviceVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, versionType, + policy != null ? policy.getZoneName() : null, policyDeltaType, policy); transactionSynchronizationAdapter.executeOnTransactionCommit(serviceVersionUpdater); if (updateServiceInfoRoleVersion) { - Runnable roleVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, VERSION_TYPE.ROLE_VERSION, policy != null ? policy.getZoneName() : null, policyDeltaType, policy); + Runnable roleVersionUpdater = new ServiceVersionUpdater(daoManager, serviceId, VERSION_TYPE.ROLE_VERSION, + policy != null ? policy.getZoneName() : null, policyDeltaType, policy); transactionSynchronizationAdapter.executeOnTransactionCommit(roleVersionUpdater); } } private boolean isRoleDownloadRequired(RangerPolicy policy, RangerService service) { - // Role Download to plugin is required if some role in the policy created/updated is not present in any other + // Role Download to plugin is required if some role in the policy + // created/updated is not present in any other // policy for that service. boolean ret = false; @@ -4394,16 +4842,17 @@ private boolean isRoleDownloadRequired(RangerPolicy policy, RangerService servic } private void checkAndFilterRoleNames(Set roleNames, RangerService service) { - //remove all roles which are already in DB for this serviceId, so we just download roles if there are new roles added. - Set rolesToRemove = new HashSet<>(); - Long serviceId = service.getId(); - List rolesFromDb = daoMgr.getXXRole().findRoleNamesByServiceId(serviceId); + // remove all roles which are already in DB for this serviceId, so we just + // download roles if there are new roles added. + Set rolesToRemove = new HashSet<>(); + Long serviceId = service.getId(); + List rolesFromDb = daoMgr.getXXRole().findRoleNamesByServiceId(serviceId); if (CollectionUtils.isNotEmpty(rolesFromDb)) { rolesToRemove.addAll(rolesFromDb); } - String tagService = service.getTagService(); + String tagService = service.getTagService(); XXService serviceDbObj = daoMgr.getXXService().findByName(tagService); if (serviceDbObj != null) { @@ -4418,7 +4867,7 @@ private void checkAndFilterRoleNames(Set roleNames, RangerService servic } private Set getAllPolicyItemRoleNames(RangerPolicy policy) { - Set ret = new HashSet<>(); + Set ret = new HashSet<>(); List policyItems = policy.getPolicyItems(); if (CollectionUtils.isNotEmpty(policyItems)) { @@ -4458,7 +4907,8 @@ private Set getAllPolicyItemRoleNames(RangerPolicy policy) { return ret; } - private void collectRolesFromPolicyItems(List rangerPolicyItems, Set roleNames) { + private void collectRolesFromPolicyItems(List rangerPolicyItems, + Set roleNames) { for (RangerPolicyItem rangerPolicyItem : rangerPolicyItems) { List rangerPolicyItemRoles = rangerPolicyItem.getRoles(); @@ -4469,18 +4919,22 @@ private void collectRolesFromPolicyItems(List ranger } private void persistChangeLog(ServiceVersionUpdater serviceVersionUpdater) { - XXServiceVersionInfoDao serviceVersionInfoDao = serviceVersionUpdater.daoManager.getXXServiceVersionInfo(); - XXServiceVersionInfo serviceVersionInfoDbObj = serviceVersionInfoDao.findByServiceId(serviceVersionUpdater.serviceId); - XXService service = serviceVersionUpdater.daoManager.getXXService().getById(serviceVersionUpdater.serviceId); + XXServiceVersionInfoDao serviceVersionInfoDao = serviceVersionUpdater.daoManager.getXXServiceVersionInfo(); + XXServiceVersionInfo serviceVersionInfoDbObj = serviceVersionInfoDao + .findByServiceId(serviceVersionUpdater.serviceId); + XXService service = serviceVersionUpdater.daoManager.getXXService().getById(serviceVersionUpdater.serviceId); if (service != null) { - Long version = serviceVersionUpdater.versionType == VERSION_TYPE.TAG_VERSION ? serviceVersionInfoDbObj.getTagVersion() : serviceVersionInfoDbObj.getPolicyVersion(); + Long version = serviceVersionUpdater.versionType == VERSION_TYPE.TAG_VERSION + ? serviceVersionInfoDbObj.getTagVersion() + : serviceVersionInfoDbObj.getPolicyVersion(); persistChangeLog(service, serviceVersionUpdater.versionType, version, serviceVersionUpdater); } } - private static void persistChangeLog(XXService service, VERSION_TYPE versionType, Long version, ServiceVersionUpdater serviceVersionUpdater) { + private static void persistChangeLog(XXService service, VERSION_TYPE versionType, Long version, + ServiceVersionUpdater serviceVersionUpdater) { Date now = new Date(); if (versionType == VERSION_TYPE.TAG_VERSION) { @@ -4488,9 +4942,9 @@ private static void persistChangeLog(XXService service, VERSION_TYPE versionType if (tagChangeType == ServiceTags.TagsChangeType.RANGER_ADMIN_START || TagDBStore.isSupportsTagDeltas()) { // Build and save TagChangeLog - XXTagChangeLog tagChangeLog = new XXTagChangeLog(); - Long serviceResourceId = serviceVersionUpdater.resourceId; - Long tagId = serviceVersionUpdater.tagId; + XXTagChangeLog tagChangeLog = new XXTagChangeLog(); + Long serviceResourceId = serviceVersionUpdater.resourceId; + Long tagId = serviceVersionUpdater.tagId; tagChangeLog.setCreateTime(now); tagChangeLog.setServiceId(service.getId()); @@ -4534,7 +4988,7 @@ private Boolean deleteExistingPolicyLabel(RangerPolicy policy) { } List xxPolicyLabelMaps = daoMgr.getXXPolicyLabelMap().findByPolicyId(policy.getId()); - XXPolicyLabelMapDao policyLabelMapDao = daoMgr.getXXPolicyLabelMap(); + XXPolicyLabelMapDao policyLabelMapDao = daoMgr.getXXPolicyLabelMap(); for (XXPolicyLabelMap xxPolicyLabelMap : xxPolicyLabelMaps) { policyLabelMapDao.remove(xxPolicyLabelMap); @@ -4577,7 +5031,8 @@ private boolean isResourceInList(String resource, List xResourceD return false; } - private void writeExcel(List policies, String excelFileName, HttpServletResponse response) throws IOException { + private void writeExcel(List policies, String excelFileName, HttpServletResponse response) + throws IOException { OutputStream outStream = null; try (Workbook workbook = new HSSFWorkbook()) { @@ -4591,13 +5046,13 @@ private void writeExcel(List policies, String excelFileName, HttpS Map svcNameToSvcType = new HashMap<>(); for (RangerPolicy policy : policies) { - List policyItems = policy.getPolicyItems(); + List policyItems = policy.getPolicyItems(); List rowFilterPolicyItems = policy.getRowFilterPolicyItems(); - List dataMaskPolicyItems = policy.getDataMaskPolicyItems(); - List allowExceptions = policy.getAllowExceptions(); - List denyExceptions = policy.getDenyExceptions(); - List denyPolicyItems = policy.getDenyPolicyItems(); - String serviceType = policy.getServiceType(); + List dataMaskPolicyItems = policy.getDataMaskPolicyItems(); + List allowExceptions = policy.getAllowExceptions(); + List denyExceptions = policy.getDenyExceptions(); + List denyPolicyItems = policy.getDenyPolicyItems(); + String serviceType = policy.getServiceType(); if (StringUtils.isBlank(serviceType)) { serviceType = svcNameToSvcType.get(policy.getService()); @@ -4615,39 +5070,45 @@ private void writeExcel(List policies, String excelFileName, HttpS for (RangerPolicyItem policyItem : policyItems) { Row row = sheet.createRow(++rowCount); - writeBookForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, row, POLICY_ALLOW_INCLUDE); + writeBookForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, row, + POLICY_ALLOW_INCLUDE); } } else if (CollectionUtils.isNotEmpty(dataMaskPolicyItems)) { for (RangerDataMaskPolicyItem dataMaskPolicyItem : dataMaskPolicyItems) { Row row = sheet.createRow(++rowCount); - writeBookForPolicyItems(svcNameToSvcType, policy, null, dataMaskPolicyItem, null, row, null); + writeBookForPolicyItems(svcNameToSvcType, policy, null, dataMaskPolicyItem, null, row, + null); } } else if (CollectionUtils.isNotEmpty(rowFilterPolicyItems)) { for (RangerRowFilterPolicyItem rowFilterPolicyItem : rowFilterPolicyItems) { Row row = sheet.createRow(++rowCount); - writeBookForPolicyItems(svcNameToSvcType, policy, null, null, rowFilterPolicyItem, row, null); + writeBookForPolicyItems(svcNameToSvcType, policy, null, null, rowFilterPolicyItem, row, + null); } } else if (serviceType.equalsIgnoreCase(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { if (CollectionUtils.isEmpty(policyItems)) { - Row row = sheet.createRow(++rowCount); + Row row = sheet.createRow(++rowCount); RangerPolicyItem policyItem = new RangerPolicyItem(); - writeBookForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, row, POLICY_ALLOW_INCLUDE); + writeBookForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, row, + POLICY_ALLOW_INCLUDE); } } else if (CollectionUtils.isEmpty(policyItems)) { - Row row = sheet.createRow(++rowCount); + Row row = sheet.createRow(++rowCount); RangerPolicyItem policyItem = new RangerPolicyItem(); - writeBookForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, row, POLICY_ALLOW_INCLUDE); + writeBookForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, row, + POLICY_ALLOW_INCLUDE); } if (CollectionUtils.isNotEmpty(allowExceptions)) { for (RangerPolicyItem policyItem : allowExceptions) { Row row = sheet.createRow(++rowCount); - writeBookForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, row, POLICY_ALLOW_EXCLUDE); + writeBookForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, row, + POLICY_ALLOW_EXCLUDE); } } @@ -4655,7 +5116,8 @@ private void writeExcel(List policies, String excelFileName, HttpS for (RangerPolicyItem policyItem : denyExceptions) { Row row = sheet.createRow(++rowCount); - writeBookForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, row, POLICY_DENY_EXCLUDE); + writeBookForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, row, + POLICY_DENY_EXCLUDE); } } @@ -4663,7 +5125,8 @@ private void writeExcel(List policies, String excelFileName, HttpS for (RangerPolicyItem policyItem : denyPolicyItems) { Row row = sheet.createRow(++rowCount); - writeBookForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, row, POLICY_DENY_INCLUDE); + writeBookForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, row, + POLICY_DENY_INCLUDE); } } } @@ -4708,13 +5171,13 @@ private StringBuilder writeCSV(List policies, String cSVFileName, Map svcNameToSvcType = new HashMap<>(); for (RangerPolicy policy : policies) { - List policyItems = policy.getPolicyItems(); + List policyItems = policy.getPolicyItems(); List rowFilterPolicyItems = policy.getRowFilterPolicyItems(); - List dataMaskPolicyItems = policy.getDataMaskPolicyItems(); - List allowExceptions = policy.getAllowExceptions(); - List denyExceptions = policy.getDenyExceptions(); - List denyPolicyItems = policy.getDenyPolicyItems(); - String serviceType = policy.getServiceType(); + List dataMaskPolicyItems = policy.getDataMaskPolicyItems(); + List allowExceptions = policy.getAllowExceptions(); + List denyExceptions = policy.getDenyExceptions(); + List denyPolicyItems = policy.getDenyPolicyItems(); + String serviceType = policy.getServiceType(); if (StringUtils.isBlank(serviceType)) { serviceType = svcNameToSvcType.get(policy.getService()); @@ -4730,41 +5193,49 @@ private StringBuilder writeCSV(List policies, String cSVFileName, if (CollectionUtils.isNotEmpty(policyItems)) { for (RangerPolicyItem policyItem : policyItems) { - writeCSVForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, csvBuffer, POLICY_ALLOW_INCLUDE); + writeCSVForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, csvBuffer, + POLICY_ALLOW_INCLUDE); } } else if (CollectionUtils.isNotEmpty(dataMaskPolicyItems)) { for (RangerDataMaskPolicyItem dataMaskPolicyItem : dataMaskPolicyItems) { - writeCSVForPolicyItems(svcNameToSvcType, policy, null, dataMaskPolicyItem, null, csvBuffer, null); + writeCSVForPolicyItems(svcNameToSvcType, policy, null, dataMaskPolicyItem, null, csvBuffer, + null); } } else if (CollectionUtils.isNotEmpty(rowFilterPolicyItems)) { for (RangerRowFilterPolicyItem rowFilterPolicyItem : rowFilterPolicyItems) { - writeCSVForPolicyItems(svcNameToSvcType, policy, null, null, rowFilterPolicyItem, csvBuffer, null); + writeCSVForPolicyItems(svcNameToSvcType, policy, null, null, rowFilterPolicyItem, csvBuffer, + null); } } else if (serviceType.equalsIgnoreCase(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { if (CollectionUtils.isEmpty(policyItems)) { RangerPolicyItem policyItem = new RangerPolicyItem(); - writeCSVForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, csvBuffer, POLICY_ALLOW_INCLUDE); + writeCSVForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, csvBuffer, + POLICY_ALLOW_INCLUDE); } } else if (CollectionUtils.isEmpty(policyItems)) { RangerPolicyItem policyItem = new RangerPolicyItem(); - writeCSVForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, csvBuffer, POLICY_ALLOW_INCLUDE); + writeCSVForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, csvBuffer, + POLICY_ALLOW_INCLUDE); } if (CollectionUtils.isNotEmpty(allowExceptions)) { for (RangerPolicyItem policyItem : allowExceptions) { - writeCSVForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, csvBuffer, POLICY_ALLOW_EXCLUDE); + writeCSVForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, csvBuffer, + POLICY_ALLOW_EXCLUDE); } } if (CollectionUtils.isNotEmpty(denyExceptions)) { for (RangerPolicyItem policyItem : denyExceptions) { - writeCSVForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, csvBuffer, POLICY_DENY_EXCLUDE); + writeCSVForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, csvBuffer, + POLICY_DENY_EXCLUDE); } } if (CollectionUtils.isNotEmpty(denyPolicyItems)) { for (RangerPolicyItem policyItem : denyPolicyItems) { - writeCSVForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, csvBuffer, POLICY_DENY_INCLUDE); + writeCSVForPolicyItems(svcNameToSvcType, policy, policyItem, null, null, csvBuffer, + POLICY_DENY_INCLUDE); } } } @@ -4776,81 +5247,85 @@ private StringBuilder writeCSV(List policies, String cSVFileName, return csvBuffer; } - private void writeCSVForPolicyItems(Map svcNameToSvcType, RangerPolicy policy, RangerPolicyItem policyItem, RangerDataMaskPolicyItem dataMaskPolicyItem, RangerRowFilterPolicyItem rowFilterPolicyItem, StringBuilder csvBuffer, String policyConditionType) { + private void writeCSVForPolicyItems(Map svcNameToSvcType, RangerPolicy policy, + RangerPolicyItem policyItem, RangerDataMaskPolicyItem dataMaskPolicyItem, + RangerRowFilterPolicyItem rowFilterPolicyItem, StringBuilder csvBuffer, String policyConditionType) { LOG.debug("policyConditionType:[{}]", policyConditionType); - List roles = new ArrayList<>(); - List groups = new ArrayList<>(); - List users = new ArrayList<>(); - String roleNames = ""; - String groupNames = ""; - String userNames = ""; - String policyLabelName = ""; - String accessType = ""; - Boolean delegateAdmin = false; - String isExcludesValue = ""; - String maskingInfo = ""; - List accesses = new ArrayList<>(); - List conditionsList = new ArrayList<>(); - String conditionKeyValue = ""; - String resourceKeyVal = ""; - String isRecursiveValue = ""; - String serviceType = ""; - String filterExpr = ""; - String policyConditionTypeValue = ""; - String serviceName = policy.getService(); - String description = policy.getDescription(); - Boolean isAuditEnabled = policy.getIsAuditEnabled(); - List policyLabels = policy.getPolicyLabels(); - StringBuilder sb = new StringBuilder(); - StringBuilder sbIsRecursive = new StringBuilder(); - StringBuilder sbIsExcludes = new StringBuilder(); - Map resources = policy.getResources(); - String policyName = policy.getName(); + List roles = new ArrayList<>(); + List groups = new ArrayList<>(); + List users = new ArrayList<>(); + String roleNames = ""; + String groupNames = ""; + String userNames = ""; + String policyLabelName = ""; + String accessType = ""; + Boolean delegateAdmin = false; + String isExcludesValue = ""; + String maskingInfo = ""; + List accesses = new ArrayList<>(); + List conditionsList = new ArrayList<>(); + String conditionKeyValue = ""; + String resourceKeyVal = ""; + String isRecursiveValue = ""; + String serviceType = ""; + String filterExpr = ""; + String policyConditionTypeValue = ""; + String serviceName = policy.getService(); + String description = policy.getDescription(); + Boolean isAuditEnabled = policy.getIsAuditEnabled(); + List policyLabels = policy.getPolicyLabels(); + StringBuilder sb = new StringBuilder(); + StringBuilder sbIsRecursive = new StringBuilder(); + StringBuilder sbIsExcludes = new StringBuilder(); + Map resources = policy.getResources(); + String policyName = policy.getName(); policyName = policyName.replace("|", ""); if (resources != null) { for (Entry resource : resources.entrySet()) { - String resKey = resource.getKey(); + String resKey = resource.getKey(); RangerPolicyResource policyResource = resource.getValue(); - List resvalueList = policyResource.getValues(); - String isExcludes = policyResource.getIsExcludes().toString(); - String isRecursive = policyResource.getIsRecursive().toString(); - String resValue = resvalueList.toString(); + List resvalueList = policyResource.getValues(); + String isExcludes = policyResource.getIsExcludes().toString(); + String isRecursive = policyResource.getIsRecursive().toString(); + String resValue = resvalueList.toString(); sb.append(resourceKeyVal).append(" ").append(resKey).append("=").append(resValue); - sbIsExcludes.append(resourceKeyVal).append(" ").append(resKey).append("=[").append(isExcludes).append("]"); - sbIsRecursive.append(resourceKeyVal).append(" ").append(resKey).append("=[").append(isRecursive).append("]"); + sbIsExcludes.append(resourceKeyVal).append(" ").append(resKey).append("=[").append(isExcludes) + .append("]"); + sbIsRecursive.append(resourceKeyVal).append(" ").append(resKey).append("=[").append(isRecursive) + .append("]"); } - isExcludesValue = sbIsExcludes.toString(); - isExcludesValue = isExcludesValue.substring(1); + isExcludesValue = sbIsExcludes.toString(); + isExcludesValue = isExcludesValue.substring(1); isRecursiveValue = sbIsRecursive.toString(); isRecursiveValue = isRecursiveValue.substring(1); - resourceKeyVal = sb.toString(); - resourceKeyVal = resourceKeyVal.substring(1); + resourceKeyVal = sb.toString(); + resourceKeyVal = resourceKeyVal.substring(1); if (policyItem != null && dataMaskPolicyItem == null && rowFilterPolicyItem == null) { - roles = policyItem.getRoles(); - groups = policyItem.getGroups(); - users = policyItem.getUsers(); - accesses = policyItem.getAccesses(); - delegateAdmin = policyItem.getDelegateAdmin(); + roles = policyItem.getRoles(); + groups = policyItem.getGroups(); + users = policyItem.getUsers(); + accesses = policyItem.getAccesses(); + delegateAdmin = policyItem.getDelegateAdmin(); conditionsList = policyItem.getConditions(); } else if (dataMaskPolicyItem != null && policyItem == null && rowFilterPolicyItem == null) { - roles = dataMaskPolicyItem.getRoles(); - groups = dataMaskPolicyItem.getGroups(); - users = dataMaskPolicyItem.getUsers(); - accesses = dataMaskPolicyItem.getAccesses(); - delegateAdmin = dataMaskPolicyItem.getDelegateAdmin(); + roles = dataMaskPolicyItem.getRoles(); + groups = dataMaskPolicyItem.getGroups(); + users = dataMaskPolicyItem.getUsers(); + accesses = dataMaskPolicyItem.getAccesses(); + delegateAdmin = dataMaskPolicyItem.getDelegateAdmin(); conditionsList = dataMaskPolicyItem.getConditions(); RangerPolicy.RangerPolicyItemDataMaskInfo dataMaskInfo = dataMaskPolicyItem.getDataMaskInfo(); - String dataMaskType = dataMaskInfo.getDataMaskType(); + String dataMaskType = dataMaskInfo.getDataMaskType(); String conditionExpr = dataMaskInfo.getConditionExpr(); - String valueExpr = dataMaskInfo.getValueExpr(); + String valueExpr = dataMaskInfo.getValueExpr(); maskingInfo = "dataMasktype=[" + dataMaskType + "]"; @@ -4858,11 +5333,11 @@ private void writeCSVForPolicyItems(Map svcNameToSvcType, Ranger maskingInfo = maskingInfo + "; conditionExpr=[" + conditionExpr + "]"; } } else if (rowFilterPolicyItem != null && policyItem == null && dataMaskPolicyItem == null) { - roles = rowFilterPolicyItem.getRoles(); - groups = rowFilterPolicyItem.getGroups(); - users = rowFilterPolicyItem.getUsers(); - accesses = rowFilterPolicyItem.getAccesses(); - delegateAdmin = rowFilterPolicyItem.getDelegateAdmin(); + roles = rowFilterPolicyItem.getRoles(); + groups = rowFilterPolicyItem.getGroups(); + users = rowFilterPolicyItem.getUsers(); + accesses = rowFilterPolicyItem.getAccesses(); + delegateAdmin = rowFilterPolicyItem.getDelegateAdmin(); conditionsList = rowFilterPolicyItem.getConditions(); RangerPolicy.RangerPolicyItemRowFilterInfo filterInfo = rowFilterPolicyItem.getRowFilterInfo(); @@ -4885,8 +5360,8 @@ private void writeCSVForPolicyItems(Map svcNameToSvcType, Ranger if (CollectionUtils.isNotEmpty(roles)) { for (String role : roles) { if (StringUtils.isNotBlank(role)) { - role = role.replace("|", ""); - role = role.replace("#", ""); + role = role.replace("|", ""); + role = role.replace("#", ""); roleNames = roleNames + role + "#"; } } @@ -4899,8 +5374,8 @@ private void writeCSVForPolicyItems(Map svcNameToSvcType, Ranger if (CollectionUtils.isNotEmpty(groups)) { for (String group : groups) { if (StringUtils.isNotBlank(group)) { - group = group.replace("|", ""); - group = group.replace("#", ""); + group = group.replace("|", ""); + group = group.replace("#", ""); groupNames = groupNames + group + "#"; } } @@ -4913,8 +5388,8 @@ private void writeCSVForPolicyItems(Map svcNameToSvcType, Ranger if (CollectionUtils.isNotEmpty(users)) { for (String user : users) { if (StringUtils.isNotBlank(user)) { - user = user.replace("|", ""); - user = user.replace("#", ""); + user = user.replace("|", ""); + user = user.replace("#", ""); userNames = userNames + user + "#"; } } @@ -4925,9 +5400,9 @@ private void writeCSVForPolicyItems(Map svcNameToSvcType, Ranger } for (RangerPolicyItemCondition conditions : conditionsList) { - String conditionType = conditions.getType(); - List conditionList = conditions.getValues(); - String conditionValue = conditionList.toString(); + String conditionType = conditions.getType(); + List conditionList = conditions.getValues(); + String conditionValue = conditionList.toString(); conditionKeyValue = conditionType + "=" + conditionValue; } @@ -4956,8 +5431,8 @@ private void writeCSVForPolicyItems(Map svcNameToSvcType, Ranger if (CollectionUtils.isNotEmpty(policyLabels)) { for (String policyLabel : policyLabels) { if (StringUtils.isNotBlank(policyLabel)) { - policyLabel = policyLabel.replace("|", ""); - policyLabel = policyLabel.replace("#", ""); + policyLabel = policyLabel.replace("|", ""); + policyLabel = policyLabel.replace("#", ""); policyLabelName = policyLabelName + policyLabel + "#"; } } @@ -5013,15 +5488,17 @@ private void writeCSVForPolicyItems(Map svcNameToSvcType, Ranger } private String sanitizeCell(String value) { - return (value != null && !value.isEmpty() && CSV_SANITIZATION_PATTERN.matcher(value).find()) ? " " + value : value; + return (value != null && !value.isEmpty() && CSV_SANITIZATION_PATTERN.matcher(value).find()) ? " " + value + : value; } - private void writeJson(List objList, String jsonFileName, HttpServletResponse response, JSON_FILE_NAME_TYPE type) { + private void writeJson(List objList, String jsonFileName, HttpServletResponse response, + JSON_FILE_NAME_TYPE type) { response.setContentType("text/json"); response.setHeader("Content-Disposition", "attachment; filename=" + jsonFileName); ServletOutputStream out = null; - String json; + String json; switch (type) { case POLICY: @@ -5069,37 +5546,39 @@ private void writeJson(List objList, String jsonFileName, HttpServletResp } } - private void writeBookForPolicyItems(Map svcNameToSvcType, RangerPolicy policy, RangerPolicyItem policyItem, RangerDataMaskPolicyItem dataMaskPolicyItem, RangerRowFilterPolicyItem rowFilterPolicyItem, Row row, String policyConditionType) { + private void writeBookForPolicyItems(Map svcNameToSvcType, RangerPolicy policy, + RangerPolicyItem policyItem, RangerDataMaskPolicyItem dataMaskPolicyItem, + RangerRowFilterPolicyItem rowFilterPolicyItem, Row row, String policyConditionType) { LOG.debug("policyConditionType:[{}]", policyConditionType); - List groups = new ArrayList<>(); - List users = new ArrayList<>(); - List roles = new ArrayList<>(); - String roleNames = ""; - String groupNames = ""; - String policyConditionTypeValue = ""; - String userNames = ""; - String policyLabelNames = ""; - String accessType = ""; - Boolean delegateAdmin = false; - String isRecursive; - String isExcludes; - Boolean isAuditEnabled = policy.getIsAuditEnabled(); - String isExcludesValue = ""; - - List accesses = new ArrayList<>(); - List conditionsList = new ArrayList<>(); - String conditionKeyValue = ""; - List policyLabels; - String resValue; - String resourceKeyVal = ""; - String isRecursiveValue = ""; - String resKey; - StringBuilder sb = new StringBuilder(); - StringBuilder sbIsRecursive = new StringBuilder(); - StringBuilder sbIsExcludes = new StringBuilder(); - Map resources = policy.getResources(); - RangerPolicy.RangerPolicyItemDataMaskInfo dataMaskInfo; + List groups = new ArrayList<>(); + List users = new ArrayList<>(); + List roles = new ArrayList<>(); + String roleNames = ""; + String groupNames = ""; + String policyConditionTypeValue = ""; + String userNames = ""; + String policyLabelNames = ""; + String accessType = ""; + Boolean delegateAdmin = false; + String isRecursive; + String isExcludes; + Boolean isAuditEnabled = policy.getIsAuditEnabled(); + String isExcludesValue = ""; + + List accesses = new ArrayList<>(); + List conditionsList = new ArrayList<>(); + String conditionKeyValue = ""; + List policyLabels; + String resValue; + String resourceKeyVal = ""; + String isRecursiveValue = ""; + String resKey; + StringBuilder sb = new StringBuilder(); + StringBuilder sbIsRecursive = new StringBuilder(); + StringBuilder sbIsExcludes = new StringBuilder(); + Map resources = policy.getResources(); + RangerPolicy.RangerPolicyItemDataMaskInfo dataMaskInfo; RangerPolicy.RangerPolicyItemRowFilterInfo filterInfo; row.createCell(0).setCellValue(policy.getId()); @@ -5110,46 +5589,48 @@ private void writeBookForPolicyItems(Map svcNameToSvcType, Range resKey = resource.getKey(); RangerPolicyResource policyResource = resource.getValue(); - List resvalueList = policyResource.getValues(); + List resvalueList = policyResource.getValues(); - isExcludes = policyResource.getIsExcludes().toString(); + isExcludes = policyResource.getIsExcludes().toString(); isRecursive = policyResource.getIsRecursive().toString(); - resValue = resvalueList.toString(); + resValue = resvalueList.toString(); sb.append(resourceKeyVal).append("; ").append(resKey).append("=").append(resValue); - sbIsExcludes.append(resourceKeyVal).append("; ").append(resKey).append("=[").append(isExcludes).append("]"); - sbIsRecursive.append(resourceKeyVal).append("; ").append(resKey).append("=[").append(isRecursive).append("]"); + sbIsExcludes.append(resourceKeyVal).append("; ").append(resKey).append("=[").append(isExcludes) + .append("]"); + sbIsRecursive.append(resourceKeyVal).append("; ").append(resKey).append("=[").append(isRecursive) + .append("]"); } - isExcludesValue = sbIsExcludes.toString(); - isExcludesValue = isExcludesValue.substring(1); + isExcludesValue = sbIsExcludes.toString(); + isExcludesValue = isExcludesValue.substring(1); isRecursiveValue = sbIsRecursive.toString(); isRecursiveValue = isRecursiveValue.substring(1); - resourceKeyVal = sb.toString(); - resourceKeyVal = resourceKeyVal.substring(1); + resourceKeyVal = sb.toString(); + resourceKeyVal = resourceKeyVal.substring(1); row.createCell(2).setCellValue(sanitizeCell(resourceKeyVal)); if (policyItem != null && dataMaskPolicyItem == null && rowFilterPolicyItem == null) { - roles = policyItem.getRoles(); - groups = policyItem.getGroups(); - users = policyItem.getUsers(); - accesses = policyItem.getAccesses(); - delegateAdmin = policyItem.getDelegateAdmin(); + roles = policyItem.getRoles(); + groups = policyItem.getGroups(); + users = policyItem.getUsers(); + accesses = policyItem.getAccesses(); + delegateAdmin = policyItem.getDelegateAdmin(); conditionsList = policyItem.getConditions(); } else if (dataMaskPolicyItem != null && policyItem == null && rowFilterPolicyItem == null) { - roles = dataMaskPolicyItem.getRoles(); - groups = dataMaskPolicyItem.getGroups(); - users = dataMaskPolicyItem.getUsers(); - accesses = dataMaskPolicyItem.getAccesses(); - delegateAdmin = dataMaskPolicyItem.getDelegateAdmin(); + roles = dataMaskPolicyItem.getRoles(); + groups = dataMaskPolicyItem.getGroups(); + users = dataMaskPolicyItem.getUsers(); + accesses = dataMaskPolicyItem.getAccesses(); + delegateAdmin = dataMaskPolicyItem.getDelegateAdmin(); conditionsList = dataMaskPolicyItem.getConditions(); - dataMaskInfo = dataMaskPolicyItem.getDataMaskInfo(); + dataMaskInfo = dataMaskPolicyItem.getDataMaskInfo(); - String dataMaskType = dataMaskInfo.getDataMaskType(); + String dataMaskType = dataMaskInfo.getDataMaskType(); String conditionExpr = dataMaskInfo.getConditionExpr(); - String valueExpr = dataMaskInfo.getValueExpr(); - String maskingInfo = "dataMasktype=[" + dataMaskType + "]"; + String valueExpr = dataMaskInfo.getValueExpr(); + String maskingInfo = "dataMasktype=[" + dataMaskType + "]"; if (conditionExpr != null && !conditionExpr.isEmpty() && valueExpr != null && !valueExpr.isEmpty()) { maskingInfo = maskingInfo + "; conditionExpr=[" + conditionExpr + "]"; @@ -5157,13 +5638,13 @@ private void writeBookForPolicyItems(Map svcNameToSvcType, Range row.createCell(18).setCellValue(sanitizeCell(maskingInfo)); } else if (rowFilterPolicyItem != null && policyItem == null && dataMaskPolicyItem == null) { - roles = rowFilterPolicyItem.getRoles(); - groups = rowFilterPolicyItem.getGroups(); - users = rowFilterPolicyItem.getUsers(); - accesses = rowFilterPolicyItem.getAccesses(); - delegateAdmin = rowFilterPolicyItem.getDelegateAdmin(); + roles = rowFilterPolicyItem.getRoles(); + groups = rowFilterPolicyItem.getGroups(); + users = rowFilterPolicyItem.getUsers(); + accesses = rowFilterPolicyItem.getAccesses(); + delegateAdmin = rowFilterPolicyItem.getDelegateAdmin(); conditionsList = rowFilterPolicyItem.getConditions(); - filterInfo = rowFilterPolicyItem.getRowFilterInfo(); + filterInfo = rowFilterPolicyItem.getRowFilterInfo(); String filterExpr = filterInfo.getFilterExpr(); @@ -5211,10 +5692,10 @@ private void writeBookForPolicyItems(Map svcNameToSvcType, Range String conditionValue = ""; for (RangerPolicyItemCondition conditions : conditionsList) { - String conditionType = conditions.getType(); + String conditionType = conditions.getType(); List conditionList = conditions.getValues(); - conditionValue = conditionList.toString(); + conditionValue = conditionList.toString(); conditionKeyValue = conditionType + "=" + conditionValue; } @@ -5287,7 +5768,7 @@ private String getPolicyTypeString(int policyType) { private void createHeaderRow(Sheet sheet) { CellStyle cellStyle = sheet.getWorkbook().createCellStyle(); - Font font = sheet.getWorkbook().createFont(); + Font font = sheet.getWorkbook().createFont(); font.setBold(true); font.setFontHeightInPoints((short) 12); @@ -5381,14 +5862,14 @@ private void createHeaderRow(Sheet sheet) { } private RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) { - List policyList = new ArrayList<>(); - RangerPolicyList retList = new RangerPolicyList(); - Map policyMap = new HashMap<>(); - Set processedServices = new HashSet<>(); - Set processedSvcIdsForRole = new HashSet<>(); - Set processedPolicies = new HashSet<>(); - List xPolList = null; - String serviceName = searchFilter.getParam(ServiceREST.PARAM_SERVICE_NAME); + List policyList = new ArrayList<>(); + RangerPolicyList retList = new RangerPolicyList(); + Map policyMap = new HashMap<>(); + Set processedServices = new HashSet<>(); + Set processedSvcIdsForRole = new HashSet<>(); + Set processedPolicies = new HashSet<>(); + List xPolList = null; + String serviceName = searchFilter.getParam(ServiceREST.PARAM_SERVICE_NAME); if (StringUtils.isNotBlank(serviceName)) { Long serviceId = getRangerServiceByName(serviceName.trim()); @@ -5397,7 +5878,8 @@ private RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) { loadRangerPolicies(serviceId, processedServices, policyMap, searchFilter); } } else { - xPolList = policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, retList); + xPolList = policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, + retList); if (!CollectionUtils.isEmpty(xPolList)) { for (XXPolicy xXPolicy : xPolList) { @@ -5413,14 +5895,16 @@ private RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) { if (!StringUtils.isEmpty(userName)) { searchFilter.setParam("user", RangerPolicyEngine.USER_CURRENT); - List xPolListForMacroUser = policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, retList); - Set processedSvcIdsForMacroUser = new HashSet<>(); + List xPolListForMacroUser = policyService.searchResources(searchFilter, + policyService.searchFields, policyService.sortFields, retList); + Set processedSvcIdsForMacroUser = new HashSet<>(); if (!CollectionUtils.isEmpty(xPolListForMacroUser)) { for (XXPolicy xXPolicy : xPolListForMacroUser) { if (!processedPolicies.contains(xXPolicy.getId())) { if (!processedSvcIdsForMacroUser.contains(xXPolicy.getService())) { - loadRangerPolicies(xXPolicy.getService(), processedSvcIdsForMacroUser, policyMap, searchFilter); + loadRangerPolicies(xXPolicy.getService(), processedSvcIdsForMacroUser, policyMap, + searchFilter); } if (policyMap.get(xXPolicy.getId()) != null) { @@ -5438,20 +5922,23 @@ private RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) { groupNames.add(RangerConstants.GROUP_PUBLIC); - Set processedSvcIdsForGroup = new HashSet<>(); - Set processedGroupsName = new HashSet<>(); + Set processedSvcIdsForGroup = new HashSet<>(); + Set processedGroupsName = new HashSet<>(); for (String groupName : groupNames) { searchFilter.setParam("group", groupName); - List xPolList2 = policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, retList); + List xPolList2 = policyService.searchResources(searchFilter, policyService.searchFields, + policyService.sortFields, retList); if (!CollectionUtils.isEmpty(xPolList2)) { for (XXPolicy xPol2 : xPolList2) { if (xPol2 != null) { if (!processedPolicies.contains(xPol2.getId())) { - if (!processedSvcIdsForGroup.contains(xPol2.getService()) || !processedGroupsName.contains(groupName)) { - loadRangerPolicies(xPol2.getService(), processedSvcIdsForGroup, policyMap, searchFilter); + if (!processedSvcIdsForGroup.contains(xPol2.getService()) + || !processedGroupsName.contains(groupName)) { + loadRangerPolicies(xPol2.getService(), processedSvcIdsForGroup, policyMap, + searchFilter); processedGroupsName.add(groupName); } @@ -5473,27 +5960,30 @@ private RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) { XXUser xxUser = daoMgr.getXXUser().findByUserName(userName); if (xxUser != null) { - Set allContainedRoles = new HashSet<>(); - List xxRoles = daoMgr.getXXRole().findByUserId(xxUser.getId()); + Set allContainedRoles = new HashSet<>(); + List xxRoles = daoMgr.getXXRole().findByUserId(xxUser.getId()); for (XXRole xxRole : xxRoles) { getContainingRoles(xxRole.getId(), allContainedRoles); } - Set roleNames = getRoleNames(allContainedRoles); + Set roleNames = getRoleNames(allContainedRoles); Set processedRoleName = new HashSet<>(); for (String roleName : roleNames) { searchFilter.setParam("role", roleName); - List xPolList3 = policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, retList); + List xPolList3 = policyService.searchResources(searchFilter, policyService.searchFields, + policyService.sortFields, retList); if (!CollectionUtils.isEmpty(xPolList3)) { for (XXPolicy xPol3 : xPolList3) { if (xPol3 != null) { if (!processedPolicies.contains(xPol3.getId())) { - if (!processedSvcIdsForRole.contains(xPol3.getService()) || !processedRoleName.contains(roleName)) { - loadRangerPolicies(xPol3.getService(), processedSvcIdsForRole, policyMap, searchFilter); + if (!processedSvcIdsForRole.contains(xPol3.getService()) + || !processedRoleName.contains(roleName)) { + loadRangerPolicies(xPol3.getService(), processedSvcIdsForRole, policyMap, + searchFilter); processedRoleName.add(roleName); } @@ -5522,20 +6012,23 @@ private RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) { groupNames.add(groupName); - Set processedSvcIdsForGroup = new HashSet<>(); - Set processedGroupsName = new HashSet<>(); + Set processedSvcIdsForGroup = new HashSet<>(); + Set processedGroupsName = new HashSet<>(); for (String grpName : groupNames) { searchFilter.setParam("group", grpName); - List xPolList2 = policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, retList); + List xPolList2 = policyService.searchResources(searchFilter, policyService.searchFields, + policyService.sortFields, retList); if (!CollectionUtils.isEmpty(xPolList2)) { for (XXPolicy xPol2 : xPolList2) { if (xPol2 != null) { if (!processedPolicies.contains(xPol2.getId())) { - if (!processedSvcIdsForGroup.contains(xPol2.getService()) || !processedGroupsName.contains(groupName)) { - loadRangerPolicies(xPol2.getService(), processedSvcIdsForGroup, policyMap, searchFilter); + if (!processedSvcIdsForGroup.contains(xPol2.getService()) + || !processedGroupsName.contains(groupName)) { + loadRangerPolicies(xPol2.getService(), processedSvcIdsForGroup, policyMap, + searchFilter); processedGroupsName.add(groupName); } @@ -5556,27 +6049,30 @@ private RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) { XXGroup xxGroup = daoMgr.getXXGroup().findByGroupName(groupName); if (xxGroup != null) { - Set allContainedRoles = new HashSet<>(); - List xxRoles = daoMgr.getXXRole().findByGroupId(xxGroup.getId()); + Set allContainedRoles = new HashSet<>(); + List xxRoles = daoMgr.getXXRole().findByGroupId(xxGroup.getId()); for (XXRole xxRole : xxRoles) { getContainingRoles(xxRole.getId(), allContainedRoles); } - Set roleNames = getRoleNames(allContainedRoles); + Set roleNames = getRoleNames(allContainedRoles); Set processedRoleName = new HashSet<>(); for (String roleName : roleNames) { searchFilter.setParam("role", roleName); - List xPolList3 = policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, retList); + List xPolList3 = policyService.searchResources(searchFilter, policyService.searchFields, + policyService.sortFields, retList); if (!CollectionUtils.isEmpty(xPolList3)) { for (XXPolicy xPol3 : xPolList3) { if (xPol3 != null) { if (!processedPolicies.contains(xPol3.getId())) { - if (!processedSvcIdsForRole.contains(xPol3.getService()) || !processedRoleName.contains(roleName)) { - loadRangerPolicies(xPol3.getService(), processedSvcIdsForRole, policyMap, searchFilter); + if (!processedSvcIdsForRole.contains(xPol3.getService()) + || !processedRoleName.contains(roleName)) { + loadRangerPolicies(xPol3.getService(), processedSvcIdsForRole, policyMap, + searchFilter); processedRoleName.add(roleName); } @@ -5645,15 +6141,17 @@ private RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) { } private boolean isSearchQuerybyResource(SearchFilter searchFilter) { - boolean ret = false; - Map filterResourcesPrefix = searchFilter.getParamsWithPrefix(SearchFilter.RESOURCE_PREFIX, true); + boolean ret = false; + Map filterResourcesPrefix = searchFilter.getParamsWithPrefix(SearchFilter.RESOURCE_PREFIX, + true); if (MapUtils.isNotEmpty(filterResourcesPrefix)) { ret = true; } if (!ret) { - Map filterResourcesPolResource = searchFilter.getParamsWithPrefix(SearchFilter.POL_RESOURCE, true); + Map filterResourcesPolResource = searchFilter.getParamsWithPrefix(SearchFilter.POL_RESOURCE, + true); if (MapUtils.isNotEmpty(filterResourcesPolResource)) { ret = true; @@ -5664,7 +6162,7 @@ private boolean isSearchQuerybyResource(SearchFilter searchFilter) { } private Long getRangerServiceByName(String name) { - XXService xxService = null; + XXService xxService = null; XXServiceDao xxServiceDao = daoMgr.getXXService(); if (xxServiceDao != null) { @@ -5674,7 +6172,8 @@ private Long getRangerServiceByName(String name) { return xxService == null ? null : xxService.getId(); } - private void loadRangerPolicies(Long serviceId, Set processedServices, Map policyMap, SearchFilter searchFilter) { + private void loadRangerPolicies(Long serviceId, Set processedServices, Map policyMap, + SearchFilter searchFilter) { try { List tempPolicyList = getServicePolicies(serviceId, searchFilter); @@ -5701,12 +6200,15 @@ private void updateServiceWithCustomProperty() { List lstRangerService = getServices(filter); for (RangerService rangerService : lstRangerService) { - String serviceUser = PropertiesUtil.getProperty("ranger.plugins." + rangerService.getType() + ".serviceuser"); + String serviceUser = PropertiesUtil + .getProperty("ranger.plugins." + rangerService.getType() + ".serviceuser"); if (!StringUtils.isEmpty(serviceUser)) { boolean chkServiceUpdate = false; - LOG.debug("customproperty = {} for service = {}", rangerService.getConfigs().get(ServiceREST.Allowed_User_List_For_Download), rangerService.getName()); + LOG.debug("customproperty = {} for service = {}", + rangerService.getConfigs().get(ServiceREST.Allowed_User_List_For_Download), + rangerService.getName()); if (!rangerService.getConfigs().containsKey(ServiceREST.Allowed_User_List_For_Download)) { rangerService.getConfigs().put(ServiceREST.Allowed_User_List_For_Download, serviceUser); @@ -5714,7 +6216,9 @@ private void updateServiceWithCustomProperty() { chkServiceUpdate = true; } - if ((!rangerService.getConfigs().containsKey(ServiceREST.Allowed_User_List_For_Grant_Revoke)) && ("hbase".equalsIgnoreCase(rangerService.getType()) || "hive".equalsIgnoreCase(rangerService.getType()))) { + if ((!rangerService.getConfigs().containsKey(ServiceREST.Allowed_User_List_For_Grant_Revoke)) + && ("hbase".equalsIgnoreCase(rangerService.getType()) + || "hive".equalsIgnoreCase(rangerService.getType()))) { rangerService.getConfigs().put(ServiceREST.Allowed_User_List_For_Grant_Revoke, serviceUser); chkServiceUpdate = true; @@ -5729,7 +6233,8 @@ private void updateServiceWithCustomProperty() { if (chkServiceUpdate) { updateService(rangerService, null); - LOG.debug("Updated service {} with custom properties in secure environment", rangerService.getName()); + LOG.debug("Updated service {} with custom properties in secure environment", + rangerService.getName()); } } } @@ -5786,34 +6291,35 @@ private String getMetricOfTypeUserGroup(final SearchCriteria searchCriteria) { String ret = null; try { - VXGroupList vxGroupList = xUserMgr.searchXGroups(searchCriteria); - long groupCount = vxGroupList.getTotalCount(); + VXGroupList vxGroupList = xUserMgr.searchXGroups(searchCriteria); + long groupCount = vxGroupList.getTotalCount(); ArrayList userAdminRoleCount = new ArrayList<>(); userAdminRoleCount.add(RangerConstants.ROLE_SYS_ADMIN); - long userSysAdminCount = getUserCountBasedOnUserRole(userAdminRoleCount); + long userSysAdminCount = getUserCountBasedOnUserRole(userAdminRoleCount); ArrayList userAdminAuditorRoleCount = new ArrayList<>(); userAdminAuditorRoleCount.add(RangerConstants.ROLE_ADMIN_AUDITOR); - long userSysAdminAuditorCount = getUserCountBasedOnUserRole(userAdminAuditorRoleCount); + long userSysAdminAuditorCount = getUserCountBasedOnUserRole(userAdminAuditorRoleCount); ArrayList userRoleListKeyRoleAdmin = new ArrayList<>(); userRoleListKeyRoleAdmin.add(RangerConstants.ROLE_KEY_ADMIN); - long userKeyAdminCount = getUserCountBasedOnUserRole(userRoleListKeyRoleAdmin); + long userKeyAdminCount = getUserCountBasedOnUserRole(userRoleListKeyRoleAdmin); ArrayList userRoleListKeyadminAduitorRole = new ArrayList<>(); userRoleListKeyadminAduitorRole.add(RangerConstants.ROLE_KEY_ADMIN_AUDITOR); - long userKeyadminAuditorCount = getUserCountBasedOnUserRole(userRoleListKeyadminAduitorRole); - ArrayList userRoleListUser = new ArrayList<>(); + long userKeyadminAuditorCount = getUserCountBasedOnUserRole(userRoleListKeyadminAduitorRole); + ArrayList userRoleListUser = new ArrayList<>(); userRoleListUser.add(RangerConstants.ROLE_USER); - long userRoleCount = getUserCountBasedOnUserRole(userRoleListUser); - long userTotalCount = userSysAdminCount + userKeyAdminCount + userRoleCount + userKeyadminAuditorCount + userSysAdminAuditorCount; + long userRoleCount = getUserCountBasedOnUserRole(userRoleListUser); + long userTotalCount = userSysAdminCount + userKeyAdminCount + userRoleCount + userKeyadminAuditorCount + + userSysAdminAuditorCount; VXMetricUserGroupCount metricUserGroupCount = new VXMetricUserGroupCount(); metricUserGroupCount.setUserCountOfUserRole(userRoleCount); @@ -5826,7 +6332,8 @@ private String getMetricOfTypeUserGroup(final SearchCriteria searchCriteria) { ret = JsonUtils.objectToJson(metricUserGroupCount); } catch (Exception e) { - LOG.error("ServiceDBStore.getMetricByType(usergroup): Error calculating Metric for usergroup : {}", e.getMessage()); + LOG.error("ServiceDBStore.getMetricByType(usergroup): Error calculating Metric for usergroup : {}", + e.getMessage()); } return ret; @@ -5836,17 +6343,20 @@ private String getMetricOfTypeAudits(final SearchCriteria searchCriteria) { String ret = null; try { - int clientTimeOffsetInMinute = RestUtil.getClientTimeOffset(); - String defaultDateFormat = "MM/dd/yyyy"; - DateFormat formatter = new SimpleDateFormat(defaultDateFormat); + int clientTimeOffsetInMinute = RestUtil.getClientTimeOffset(); + String defaultDateFormat = "MM/dd/yyyy"; + DateFormat formatter = new SimpleDateFormat(defaultDateFormat); - VXMetricAuditDetailsCount auditObj = new VXMetricAuditDetailsCount(); - DateUtil dateUtilTwoDays = new DateUtil(); - Date startDateUtilTwoDays = dateUtilTwoDays.getDateFromNow(-2); + VXMetricAuditDetailsCount auditObj = new VXMetricAuditDetailsCount(); + DateUtil dateUtilTwoDays = new DateUtil(); + Date startDateUtilTwoDays = dateUtilTwoDays.getDateFromNow(-2); - Date dStart2 = restErrorUtil.parseDate(formatter.format(startDateUtilTwoDays), "Invalid value for startDate", MessageEnums.INVALID_INPUT_DATA, null, "startDate", defaultDateFormat); + Date dStart2 = restErrorUtil.parseDate(formatter.format(startDateUtilTwoDays), + "Invalid value for startDate", MessageEnums.INVALID_INPUT_DATA, null, "startDate", + defaultDateFormat); Date endDateTwoDays = MiscUtil.getUTCDate(); - Date dEnd2 = restErrorUtil.parseDate(formatter.format(endDateTwoDays), "Invalid value for endDate", MessageEnums.INVALID_INPUT_DATA, null, "endDate", defaultDateFormat); + Date dEnd2 = restErrorUtil.parseDate(formatter.format(endDateTwoDays), "Invalid value for endDate", + MessageEnums.INVALID_INPUT_DATA, null, "endDate", defaultDateFormat); dEnd2 = dateUtilTwoDays.getDateFromGivenDate(dEnd2, 0, 23, 59, 59); dEnd2 = dateUtilTwoDays.addTimeOffset(dEnd2, clientTimeOffsetInMinute); @@ -5863,13 +6373,15 @@ private String getMetricOfTypeAudits(final SearchCriteria searchCriteria) { auditObj.setSolrIndexCountTwoDays(totalAuditsCountTwoDays); - DateUtil dateUtilWeek = new DateUtil(); - Date startDateUtilWeek = dateUtilWeek.getDateFromNow(-7); - Date dStart7 = restErrorUtil.parseDate(formatter.format(startDateUtilWeek), "Invalid value for startDate", MessageEnums.INVALID_INPUT_DATA, null, "startDate", defaultDateFormat); + DateUtil dateUtilWeek = new DateUtil(); + Date startDateUtilWeek = dateUtilWeek.getDateFromNow(-7); + Date dStart7 = restErrorUtil.parseDate(formatter.format(startDateUtilWeek), "Invalid value for startDate", + MessageEnums.INVALID_INPUT_DATA, null, "startDate", defaultDateFormat); - Date endDateWeek = MiscUtil.getUTCDate(); + Date endDateWeek = MiscUtil.getUTCDate(); DateUtil dateUtilweek = new DateUtil(); - Date dEnd7 = restErrorUtil.parseDate(formatter.format(endDateWeek), "Invalid value for endDate", MessageEnums.INVALID_INPUT_DATA, null, "endDate", defaultDateFormat); + Date dEnd7 = restErrorUtil.parseDate(formatter.format(endDateWeek), "Invalid value for endDate", + MessageEnums.INVALID_INPUT_DATA, null, "endDate", defaultDateFormat); dEnd7 = dateUtilweek.getDateFromGivenDate(dEnd7, 0, 23, 59, 59); dEnd7 = dateUtilweek.addTimeOffset(dEnd7, clientTimeOffsetInMinute); @@ -5888,7 +6400,8 @@ private String getMetricOfTypeAudits(final SearchCriteria searchCriteria) { ret = JsonUtils.objectToJson(auditObj); } catch (Exception e) { - LOG.error("ServiceDBStore.getMetricByType(audits): Error calculating Metric for audits : {}", e.getMessage()); + LOG.error("ServiceDBStore.getMetricByType(audits): Error calculating Metric for audits : {}", + e.getMessage()); } return ret; @@ -5907,10 +6420,10 @@ private String getMetricOfTypeServices(final SearchCriteria searchCriteria) { serviceFilter.setSortType("asc"); VXMetricServiceCount vXMetricServiceCount = new VXMetricServiceCount(); - PList paginatedSvcs = getPaginatedServices(serviceFilter); - long totalServiceCount = paginatedSvcs.getTotalCount(); - List rangerServiceList = paginatedSvcs.getList(); - Map services = new HashMap<>(); + PList paginatedSvcs = getPaginatedServices(serviceFilter); + long totalServiceCount = paginatedSvcs.getTotalCount(); + List rangerServiceList = paginatedSvcs.getList(); + Map services = new HashMap<>(); for (RangerService rangerService : rangerServiceList) { String serviceName = rangerService.getType(); @@ -5929,7 +6442,8 @@ private String getMetricOfTypeServices(final SearchCriteria searchCriteria) { ret = JsonUtils.objectToJson(vXMetricServiceCount); } catch (Exception e) { - LOG.error("ServiceDBStore.getMetricByType(services): Error calculating Metric for services : {}", e.getMessage()); + LOG.error("ServiceDBStore.getMetricByType(services): Error calculating Metric for services : {}", + e.getMessage()); } return ret; @@ -5948,14 +6462,14 @@ private String getMetricOfTypePolicies(final SearchCriteria searchCriteria) { policyFilter.setSortType("asc"); VXMetricPolicyWithServiceNameCount vXMetricPolicyWithServiceNameCount = new VXMetricPolicyWithServiceNameCount(); - PList paginatedSvcsList = getPaginatedPolicies(policyFilter); + PList paginatedSvcsList = getPaginatedPolicies(policyFilter); vXMetricPolicyWithServiceNameCount.setTotalCount(paginatedSvcsList.getTotalCount()); Map servicesWithPolicy = new HashMap<>(); for (int k = 2; k >= 0; k--) { - String policyType = String.valueOf(k); + String policyType = String.valueOf(k); VXMetricServiceNameCount vXMetricServiceNameCount = getVXMetricServiceCount(policyType); if (k == 2) { @@ -5967,15 +6481,15 @@ private String getMetricOfTypePolicies(final SearchCriteria searchCriteria) { } } - Map> tagMap = new HashMap<>(); - Map serviceNameWithPolicyCount = new HashMap<>(); - boolean tagFlag = false; + Map> tagMap = new HashMap<>(); + Map serviceNameWithPolicyCount = new HashMap<>(); + boolean tagFlag = false; if (!tagFlag) { policyFilter.setParam("serviceType", "tag"); PList policiestype = getPaginatedPolicies(policyFilter); - List policies = policiestype.getList(); + List policies = policiestype.getList(); for (RangerPolicy rangerPolicy : policies) { if (serviceNameWithPolicyCount.containsKey(rangerPolicy.getService())) { @@ -5988,7 +6502,7 @@ private String getMetricOfTypePolicies(final SearchCriteria searchCriteria) { tagMap.put("tag", serviceNameWithPolicyCount); - long tagCount = policiestype.getTotalCount(); + long tagCount = policiestype.getTotalCount(); VXMetricServiceNameCount vXMetricServiceNameCount = new VXMetricServiceNameCount(); vXMetricServiceNameCount.setServiceBasedCountList(tagMap); @@ -6001,7 +6515,8 @@ private String getMetricOfTypePolicies(final SearchCriteria searchCriteria) { ret = JsonUtils.objectToJson(vXMetricPolicyWithServiceNameCount); } catch (Exception e) { - LOG.error("ServiceDBStore.getMetricByType(policies): Error calculating Metric for policies : {}", e.getMessage()); + LOG.error("ServiceDBStore.getMetricByType(policies): Error calculating Metric for policies : {}", + e.getMessage()); } return ret; @@ -6011,13 +6526,14 @@ private String getMetricOfTypeDatabase(final SearchCriteria searchCriteria) { String ret = null; try { - int dbFlavor = RangerBizUtil.getDBFlavor(); + int dbFlavor = RangerBizUtil.getDBFlavor(); String dbFlavourType = RangerBizUtil.getDBFlavorType(dbFlavor); - String dbDetail = dbFlavourType + " " + bizUtil.getDBVersion(); + String dbDetail = dbFlavourType + " " + bizUtil.getDBVersion(); ret = JsonUtils.objectToJson(dbDetail); } catch (Exception e) { - LOG.error("ServiceDBStore.getMetricByType(database): Error calculating Metric for database : {}", e.getMessage()); + LOG.error("ServiceDBStore.getMetricByType(database): Error calculating Metric for database : {}", + e.getMessage()); } return ret; @@ -6032,12 +6548,12 @@ private String getMetricOfTypeContextEnrichers(final SearchCriteria searchCriter filter.setStartIndex(0); VXMetricContextEnricher serviceWithContextEnrichers = new VXMetricContextEnricher(); - PList paginatedSvcDefs = getPaginatedServiceDefs(filter); - List repoTypeList = paginatedSvcDefs.getList(); + PList paginatedSvcDefs = getPaginatedServiceDefs(filter); + List repoTypeList = paginatedSvcDefs.getList(); if (repoTypeList != null) { for (RangerServiceDef repoType : repoTypeList) { - String name = repoType.getName(); + String name = repoType.getName(); List contextEnrichers = repoType.getContextEnrichers(); if (contextEnrichers != null && !contextEnrichers.isEmpty()) { @@ -6049,7 +6565,9 @@ private String getMetricOfTypeContextEnrichers(final SearchCriteria searchCriter ret = JsonUtils.objectToJson(serviceWithContextEnrichers); } catch (Exception e) { - LOG.error("ServiceDBStore.getMetricByType(contextenrichers): Error calculating Metric for contextenrichers : {}", e.getMessage()); + LOG.error( + "ServiceDBStore.getMetricByType(contextenrichers): Error calculating Metric for contextenrichers : {}", + e.getMessage()); } return ret; @@ -6068,9 +6586,9 @@ private String getMetricOfTypeDenyConditions(final SearchCriteria searchCriteria policyFilter1.setSortType("asc"); policyFilter1.setParam("denyCondition", "true"); - int denyCount = 0; - Map denyconditionsonMap = new HashMap<>(); - PList paginatedSvcDefs = getPaginatedServiceDefs(policyFilter1); + int denyCount = 0; + Map denyconditionsonMap = new HashMap<>(); + PList paginatedSvcDefs = getPaginatedServiceDefs(policyFilter1); if (paginatedSvcDefs != null) { List rangerServiceDefs = paginatedSvcDefs.getList(); @@ -6097,7 +6615,8 @@ private String getMetricOfTypeDenyConditions(final SearchCriteria searchCriteria if (policyItem != null && !policyItem.isEmpty()) { if (denyconditionsonMap.get(serviceDef) != null) { - denyCount = denyconditionsonMap.get(serviceDef) + denyCount + policyItem.size(); + denyCount = denyconditionsonMap.get(serviceDef) + denyCount + + policyItem.size(); } else { denyCount = denyCount + policyItem.size(); } @@ -6107,7 +6626,8 @@ private String getMetricOfTypeDenyConditions(final SearchCriteria searchCriteria if (policyItemExclude != null && !policyItemExclude.isEmpty()) { if (denyconditionsonMap.get(serviceDef) != null) { - denyCount = denyconditionsonMap.get(serviceDef) + denyCount + policyItemExclude.size(); + denyCount = denyconditionsonMap.get(serviceDef) + denyCount + + policyItemExclude.size(); } else { denyCount = denyCount + policyItemExclude.size(); } @@ -6130,7 +6650,9 @@ private String getMetricOfTypeDenyConditions(final SearchCriteria searchCriteria ret = JsonUtils.objectToJson(denyconditionsonMap); } catch (Exception e) { - LOG.error("ServiceDBStore.getMetricByType(denyconditions): Error calculating Metric for denyconditions : {}", e.getMessage()); + LOG.error( + "ServiceDBStore.getMetricByType(denyconditions): Error calculating Metric for denyconditions : {}", + e.getMessage()); } return ret; @@ -6146,9 +6668,9 @@ private VXMetricServiceNameCount getVXMetricServiceCount(String policyType) thro policyFilter1.setSortType("asc"); policyFilter1.setParam("policyType", policyType); - PList policies = getPaginatedPolicies(policyFilter1); - PList paginatedSvcsSevice = getPaginatedServices(policyFilter1); - List rangerServiceList = paginatedSvcsSevice.getList(); + PList policies = getPaginatedPolicies(policyFilter1); + PList paginatedSvcsSevice = getPaginatedServices(policyFilter1); + List rangerServiceList = paginatedSvcsSevice.getList(); Map> servicesforPolicyType = new HashMap<>(); long tagCount = 0; @@ -6159,9 +6681,9 @@ private VXMetricServiceNameCount getVXMetricServiceCount(String policyType) thro policyFilter1.setParam("serviceName", serviceName); - Map servicesNamewithPolicyCount = new HashMap<>(); - PList policiestype = getPaginatedPolicies(policyFilter1); - long count = policiestype.getTotalCount(); + Map servicesNamewithPolicyCount = new HashMap<>(); + PList policiestype = getPaginatedPolicies(policyFilter1); + long count = policiestype.getTotalCount(); if (count != 0) { if (!"tag".equalsIgnoreCase(servicetype)) { @@ -6194,19 +6716,19 @@ private VXMetricServiceNameCount getVXMetricServiceCount(String policyType) thro } private VXMetricServiceCount getAuditsCount(int accessResult, Date startDate, Date endDate) throws Exception { - long totalCountOfAudits = 0; - SearchFilter filter = new SearchFilter(); + long totalCountOfAudits = 0; + SearchFilter filter = new SearchFilter(); filter.setStartIndex(0); - Map servicesRepoType = new HashMap<>(); - VXMetricServiceCount vXMetricServiceCount = new VXMetricServiceCount(); - PList paginatedSvcDefs = getPaginatedServiceDefs(filter); - Iterable repoTypeGet = paginatedSvcDefs.getList(); + Map servicesRepoType = new HashMap<>(); + VXMetricServiceCount vXMetricServiceCount = new VXMetricServiceCount(); + PList paginatedSvcDefs = getPaginatedServiceDefs(filter); + Iterable repoTypeGet = paginatedSvcDefs.getList(); for (RangerServiceDef repoType : repoTypeGet) { - long id = repoType.getId(); - String serviceRepoName = repoType.getName(); + long id = repoType.getId(); + String serviceRepoName = repoType.getName(); SearchCriteria searchCriteriaWithType = new SearchCriteria(); searchCriteriaWithType.getParamList().put("repoType", id); @@ -6217,7 +6739,7 @@ private VXMetricServiceCount getAuditsCount(int accessResult, Date startDate, Da searchCriteriaWithType.setGetCount(true); VXAccessAuditList vXAccessAuditListwithType = assetMgr.getAccessLogs(searchCriteriaWithType); - long totalCountOfRepo = vXAccessAuditListwithType.getTotalCount(); + long totalCountOfRepo = vXAccessAuditListwithType.getTotalCount(); if (totalCountOfRepo != 0) { servicesRepoType.put(serviceRepoName, totalCountOfRepo); @@ -6250,12 +6772,12 @@ private Long getUserCountBasedOnUserRole(@SuppressWarnings("rawtypes") List user * @throws Exception */ private void disassociateZonesForService(RangerService service) throws Exception { - String serviceName = service.getName(); + String serviceName = service.getName(); List zonesNameList = daoMgr.getXXSecurityZoneDao().findZonesByServiceName(serviceName); if (CollectionUtils.isNotEmpty(zonesNameList)) { for (String zoneName : zonesNameList) { - RangerSecurityZone securityZone = securityZoneStore.getSecurityZoneByName(zoneName); + RangerSecurityZone securityZone = securityZoneStore.getSecurityZoneByName(zoneName); Map zoneServices = securityZone.getServices(); if (zoneServices != null && !zoneServices.isEmpty()) { @@ -6268,7 +6790,8 @@ private void disassociateZonesForService(RangerService service) throws Exception } } - private static ServicePolicies getUpdatedServicePoliciesForZones(ServicePolicies servicePolicies, Map securityZones) { + private static ServicePolicies getUpdatedServicePoliciesForZones(ServicePolicies servicePolicies, + Map securityZones) { final ServicePolicies ret; if (MapUtils.isNotEmpty(securityZones)) { @@ -6340,14 +6863,17 @@ private void patchAssociatedTagServiceInSecurityZoneInfos(ServicePolicies servic if (servicePolicies != null && MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { // Get list of zones that associated tag-service (if any) is associated with List zonesInAssociatedTagService = new ArrayList<>(); - String tagServiceName = servicePolicies.getTagPolicies() != null ? servicePolicies.getTagPolicies().getServiceName() : null; + String tagServiceName = servicePolicies.getTagPolicies() != null + ? servicePolicies.getTagPolicies().getServiceName() + : null; if (StringUtils.isNotEmpty(tagServiceName)) { try { RangerService tagService = getServiceByName(tagServiceName); if (tagService != null && tagService.getIsEnabled()) { - zonesInAssociatedTagService = daoMgr.getXXSecurityZoneDao().findZonesByTagServiceName(tagServiceName); + zonesInAssociatedTagService = daoMgr.getXXSecurityZoneDao() + .findZonesByTagServiceName(tagServiceName); } } catch (Exception exception) { LOG.warn("Could not get service associated with [{}]", tagServiceName, exception); @@ -6355,8 +6881,9 @@ private void patchAssociatedTagServiceInSecurityZoneInfos(ServicePolicies servic } if (CollectionUtils.isNotEmpty(zonesInAssociatedTagService)) { - for (Map.Entry entry : servicePolicies.getSecurityZones().entrySet()) { - String zoneName = entry.getKey(); + for (Map.Entry entry : servicePolicies.getSecurityZones() + .entrySet()) { + String zoneName = entry.getKey(); ServicePolicies.SecurityZoneInfo securityZoneInfo = entry.getValue(); securityZoneInfo.setContainsAssociatedTagService(zonesInAssociatedTagService.contains(zoneName)); @@ -6377,11 +6904,13 @@ private static List extractZonePolicies(final List a return ret; } - private static List extractZonePolicyDeltas(final List allPolicyDeltas, final String zoneName) { + private static List extractZonePolicyDeltas(final List allPolicyDeltas, + final String zoneName) { final List ret = new ArrayList<>(); for (RangerPolicyDelta delta : allPolicyDeltas) { - if (StringUtils.equals(delta.getZoneName(), zoneName) && !StringUtils.equals(delta.getServiceType(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { + if (StringUtils.equals(delta.getZoneName(), zoneName) && !StringUtils.equals(delta.getServiceType(), + EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { ret.add(delta); } } @@ -6390,9 +6919,9 @@ private static List extractZonePolicyDeltas(final List policies = servicePolicies.getPolicies(); @@ -6473,15 +7002,16 @@ private ServicePolicies filterServicePolicies(ServicePolicies servicePolicies) { } private List getAuditFiltersServiceConfigByName(String searchUsrGrpRoleName) { - LOG.debug("===> ServiceDBStore.getAuditFiltersServiceConfigByName( searchUsrGrpRoleName : {})", searchUsrGrpRoleName); + LOG.debug("===> ServiceDBStore.getAuditFiltersServiceConfigByName( searchUsrGrpRoleName : {})", + searchUsrGrpRoleName); List configMapToBeModified = null; if (StringUtils.isNotBlank(searchUsrGrpRoleName)) { configMapToBeModified = new ArrayList<>(); - XXServiceConfigMapDao configDao = daoMgr.getXXServiceConfigMap(); - List configs = configDao.findByConfigKey(ServiceDBStore.RANGER_PLUGIN_AUDIT_FILTERS); + XXServiceConfigMapDao configDao = daoMgr.getXXServiceConfigMap(); + List configs = configDao.findByConfigKey(ServiceDBStore.RANGER_PLUGIN_AUDIT_FILTERS); for (XXServiceConfigMap configMap : configs) { if (StringUtils.contains(configMap.getConfigvalue(), searchUsrGrpRoleName)) { @@ -6490,7 +7020,9 @@ private List getAuditFiltersServiceConfigByName(String searc } } - LOG.debug("<=== ServiceDBStore.getAuditFiltersServiceConfigByName( searchUsrGrpRoleName : {}) configMapToBeModified : {}", searchUsrGrpRoleName, configMapToBeModified); + LOG.debug( + "<=== ServiceDBStore.getAuditFiltersServiceConfigByName( searchUsrGrpRoleName : {}) configMapToBeModified : {}", + searchUsrGrpRoleName, configMapToBeModified); return configMapToBeModified; } @@ -6498,7 +7030,9 @@ private List getAuditFiltersServiceConfigByName(String searc private void removeUserGroupRoleReferences(List auditFilters, String user, String group, String role) { List itemsToRemove = null; - LOG.debug("===> ServiceDBStore.removeUserGroupRoleReferences( user : {} group : {} role : {} auditFilters : {})", user, group, role, auditFilters); + LOG.debug( + "===> ServiceDBStore.removeUserGroupRoleReferences( user : {} group : {} role : {} auditFilters : {})", + user, group, role, auditFilters); for (AuditFilter auditFilter : auditFilters) { boolean isAuditFilterModified = false; @@ -6521,7 +7055,9 @@ private void removeUserGroupRoleReferences(List auditFilters, Strin isAuditFilterModified = true; } - if (isAuditFilterModified && CollectionUtils.isEmpty(auditFilter.getUsers()) && CollectionUtils.isEmpty(auditFilter.getGroups()) && CollectionUtils.isEmpty(auditFilter.getRoles())) { + if (isAuditFilterModified && CollectionUtils.isEmpty(auditFilter.getUsers()) + && CollectionUtils.isEmpty(auditFilter.getGroups()) + && CollectionUtils.isEmpty(auditFilter.getRoles())) { if (itemsToRemove == null) { itemsToRemove = new ArrayList<>(); } @@ -6534,7 +7070,9 @@ private void removeUserGroupRoleReferences(List auditFilters, Strin auditFilters.removeAll(itemsToRemove); } - LOG.debug("<=== ServiceDBStore.removeUserGroupRoleReferences( user : {} group : {} role : {} auditFilters : {})", user, group, role, auditFilters); + LOG.debug( + "<=== ServiceDBStore.removeUserGroupRoleReferences( user : {} group : {} role : {} auditFilters : {})", + user, group, role, auditFilters); } private void getContainingRoles(Long roleId, Set allRoles) { @@ -6582,9 +7120,871 @@ private boolean isServiceActive(String serviceName) { return ret; } - public enum JSON_FILE_NAME_TYPE { POLICY, ROLE } + public boolean createS3BucketPolicy(RangerPolicy rangerPolicy, String action, RangerPolicy oldPolicy) + throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.createS3BucketPolicy()"); + } + String serviceName = rangerPolicy.getService(); + try { + RangerService rangerService = getServiceByName(serviceName); + Map configs = rangerService.getConfigs(); + + S3Client s3 = S3ClientConnectionMgr.getS3client(configs); + IamClient iamClient = S3ClientConnectionMgr.getIamClient(configs); + + if (s3 == null || iamClient == null) { + throw new Exception("S3 and IAM client initialization failed."); + } + + String bucketName = configs.get(RangerS3Constants.BUCKET_NAME); + + // Optimization: Extract affected buckets and filter policies + Set affectedBuckets = extractAffectedBuckets(rangerPolicy, bucketName); + SearchFilter filter = createFilterForBuckets(affectedBuckets); + List servicePolicies = getServicePolicies(serviceName, filter); + + LOG.info("Filtered policies for affected buckets {}: {} policies (instead of all policies)", + affectedBuckets, servicePolicies.size()); + + if (LOG.isDebugEnabled()) { + LOG.debug("====> createS3BucketPolicy.getServicePolicies()"); + } + + // Find the OLD version of this policy from database + RangerPolicy existingPolicyFromDB = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("====> existingPolicyFromDB old version of policies in database"); + } + + if (rangerPolicy.getId() != null && !action.equalsIgnoreCase(RangerConstants.ACTION_DELETE)) { + existingPolicyFromDB = servicePolicies.stream() + .filter(policy -> policy.getId() != null && policy.getId().equals(rangerPolicy.getId())) + .findFirst() + .orElse(null); + + if (existingPolicyFromDB != null) { + boolean policyItemsChanged = hasPolicyItemsChanged(existingPolicyFromDB, rangerPolicy); + boolean resourcesChanged = !compareResources(existingPolicyFromDB.getResources(), + rangerPolicy.getResources()); + + if (LOG.isDebugEnabled()) { + LOG.debug("Policy {} comparison - PolicyItems changed: {}, Resources changed: {}", + rangerPolicy.getId(), policyItemsChanged, resourcesChanged); + } + + if (!policyItemsChanged && !resourcesChanged) { + LOG.info("No changes detected for policy {}. Skipping S3 bucket policy update.", + rangerPolicy.getId()); + return true; + } + + LOG.info("Changes detected in policy {} - PolicyItems: {}, Resources: {}", + rangerPolicy.getId(), policyItemsChanged, resourcesChanged); + } + } else if (rangerPolicy.getId() == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("====> rangerPolicy.getId() is null"); + } + } else if (action.equalsIgnoreCase(RangerConstants.ACTION_DELETE)) { + if (LOG.isDebugEnabled()) { + LOG.debug("====> DELETE POLICY ACTION invoked"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("====> fetch old version of policy failed with reason undefined."); + } + } + + // ========== END CHANGE DETECTION ========== + + // Snapshot: capture the old resource ARNs from the policy state BEFORE the DB + // update so stale IAM statements are removed even when the resource path + // changed. + // - UPDATE: oldPolicy is read by the REST layer before svcStore.updatePolicy + // commits + // - DELETE: oldPolicy is the policy being deleted (passed before it's removed + // from DB) + // - CREATE: oldPolicy is null (nothing to clean up) + Set snapshotArns = new HashSet<>(); + if (oldPolicy != null) { + RangerPolicyResource pathResource = oldPolicy.getResources() + .get(RangerS3Constants.PATH); + if (pathResource != null && CollectionUtils.isNotEmpty(pathResource.getValues())) { + for (String s3path : pathResource.getValues()) { + snapshotArns.add(RangerS3Constants.S3_RESOURCE_PATH_ARN + s3path); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("------- Invoking combinePolicies --------"); + } + List combinedPolicies = combinePolicies(servicePolicies, rangerPolicy, action); + Map>> bucketMap = new HashMap<>(); + List affectedPolicies = populateBucketMap(bucketMap, combinedPolicies, + bucketName, rangerPolicy); + + if (affectedPolicies.isEmpty()) { + for (Entry affectedResources : rangerPolicy.getResources().entrySet()) { + List affectedResourceBuckets = affectedResources.getValue().getValues().stream() + .map(s3path -> s3path.split("/", 2)[0]) // Extract bucket name + .distinct() // Ensure unique bucket names + .collect(Collectors.toList()); + + if (affectedResourceBuckets.isEmpty()) { + deleteBucketPolicy(s3, bucketName); + } else { + for (String bucketPart : affectedResourceBuckets) { + deleteBucketPolicy(s3, bucketPart); + } + } + } + } else { + processPolicies(bucketMap, s3, iamClient, snapshotArns); + } + } catch (S3Exception e) { + throw restErrorUtil.createRESTException(e.awsErrorDetails().toString()); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.createS3BucketPolicy()"); + } + return true; + } + + private boolean hasPolicyItemsChanged(RangerPolicy oldPolicy, RangerPolicy newPolicy) { + if (LOG.isDebugEnabled()) { + LOG.debug(" ======> ServiceDBStore.hasPolicyItemsChanged()"); + } + // Compare policyItems + if (!comparePolicyItemsList(oldPolicy.getPolicyItems(), newPolicy.getPolicyItems())) { + return true; + } + + // Compare denyPolicyItems + if (!comparePolicyItemsList(oldPolicy.getDenyPolicyItems(), newPolicy.getDenyPolicyItems())) { + return true; + } + + // Compare resources + if (!compareResources(oldPolicy.getResources(), newPolicy.getResources())) { + return true; + } + + return false; + } + + private boolean comparePolicyItemsList(List list1, List list2) { + if (LOG.isDebugEnabled()) { + LOG.debug(" ======> ServiceDBStore.comparePolicyItemsList()"); + } + if (list1 == null && list2 == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("-------------- Both OLD and NEW policyItems lists are null "); + } + return true; + } + if (list1 == null || list2 == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("-------------- OLD or NEW policyItems list is null "); + } + return false; + } + if (list1.size() != list2.size()) { + if (LOG.isDebugEnabled()) { + LOG.debug("-------------- OLD and NEW policyItems list have different size "); + } + return false; + } + + for (int i = 0; i < list1.size(); i++) { + RangerPolicyItem item1 = list1.get(i); + RangerPolicyItem item2 = list2.get(i); + + if (!CollectionUtils.isEqualCollection(item1.getUsers(), item2.getUsers())) { + return false; + } + if (!CollectionUtils.isEqualCollection(item1.getGroups(), item2.getGroups())) { + return false; + } + if (!CollectionUtils.isEqualCollection(item1.getRoles(), item2.getRoles())) { + return false; + } + + // Compare access types + if (!compareAccesses(item1.getAccesses(), item2.getAccesses())) { + return false; + } + } + + return true; + } + + private boolean compareAccesses(List list1, List list2) { + if (LOG.isDebugEnabled()) { + LOG.debug(" ======> ServiceDBStore.compareAccesses()"); + } + if (list1 == null && list2 == null) { + return true; + } + if (list1 == null || list2 == null) { + return false; + } + if (list1.size() != list2.size()) { + return false; + } + + Set accessTypes1 = list1.stream().map(RangerPolicyItemAccess::getType).collect(Collectors.toSet()); + Set accessTypes2 = list2.stream().map(RangerPolicyItemAccess::getType).collect(Collectors.toSet()); + + return accessTypes1.equals(accessTypes2); + } + + private boolean compareResources(Map res1, Map res2) { + if (LOG.isDebugEnabled()) { + LOG.debug(" ======> ServiceDBStore.compareResources()"); + } + if (res1 == null && res2 == null) { + return true; + } + if (res1 == null || res2 == null) { + return false; + } + if (res1.size() != res2.size()) { + return false; + } + + for (String key : res1.keySet()) { + RangerPolicyResource resource1 = res1.get(key); + RangerPolicyResource resource2 = res2.get(key); + + if (resource2 == null) { + return false; + } + if (!CollectionUtils.isEqualCollection(resource1.getValues(), resource2.getValues())) { + return false; + } + } + + return true; + } + + /** + * Extract affected buckets from a policy's resources + * Optimization: Helps narrow down policies to only those affecting specific + * buckets + * + * @param policy Policy to extract buckets from + * @param defaultBucket Default bucket name from service config + * @return Set of affected bucket names, or null if the policy affects all + * buckets (wildcard) + */ + private Set extractAffectedBuckets(RangerPolicy policy, String defaultBucket) { + Set affectedBuckets = new HashSet<>(); + + if (policy.getResources() != null) { + for (RangerPolicyResource resource : policy.getResources().values()) { + if (resource.getValues() != null) { + for (String path : resource.getValues()) { + if (StringUtils.isEmpty(path)) { + continue; + } + if (path.startsWith("*")) { + // Wildcard affects all buckets - cannot optimize with filtering + return null; + } else { + // Extract bucket name from path (supports "bucket" or "bucket/prefix" format) + String bucket = path.split("/", 2)[0]; + affectedBuckets.add(bucket); + } + } + } + } + } + + // If no buckets found, use default bucket + if (affectedBuckets.isEmpty()) { + affectedBuckets.add(defaultBucket); + } + + return affectedBuckets; + } + + /** + * Create a SearchFilter to filter policies by affected buckets + * Optimization: Reduces O(N) scan by filtering at DB/cache level + * + * @param affectedBuckets Set of bucket names to filter by, or null for no + * filtering + * @return SearchFilter configured with bucket resource filters, or empty filter + * if optimization cannot be applied + */ + private SearchFilter createFilterForBuckets(Set affectedBuckets) { + SearchFilter filter = new SearchFilter(); + + // If null, it means wildcard - fetch all policies + if (affectedBuckets == null) { + LOG.debug("Wildcard policy detected, fetching all policies for service"); + return filter; + } + + // Use resource filtering for single bucket + // Note: SearchFilter supports resource filtering via RESOURCE_PREFIX + // For S3, we filter on "path" resource which contains bucket names or + // bucket/prefix paths + if (affectedBuckets.size() == 1) { + // Single bucket - use prefix match to get all policies affecting this bucket + // This matches "bucket", "bucket/", "bucket/path" etc. + String bucket = affectedBuckets.iterator().next(); + filter.setParam(SearchFilter.RESOURCE_PREFIX + RangerS3Constants.PATH, bucket); + LOG.debug("Filtering policies for single bucket: {}", bucket); + } else { + // Multiple buckets - SearchFilter doesn't support OR conditions across + // different resource values + // Fall back to getting all policies but still benefit from the IAM caching + // optimization + LOG.debug("Multiple affected buckets {}, fetching all policies for service", affectedBuckets); + } + + return filter; + } + + // Combine policies based on action and service policies + List combinePolicies(List servicePolicies, RangerPolicy rangerPolicy, String action) { + // Create a new list to avoid modifying the original list + List combinedPolicies = new ArrayList<>(servicePolicies); + if (LOG.isDebugEnabled()) { + LOG.debug(" ======> ServiceDBStore.combinePolicies()"); + } + + // Handle cases when servicePolicies is empty + if (servicePolicies.isEmpty()) { + if (!action.equalsIgnoreCase(RangerConstants.ACTION_DELETE)) { + return Collections.singletonList(rangerPolicy); // Add only rangerPolicy for non-delete actions + } + return combinedPolicies; // Return empty list for delete actions + } + // Check if rangerPolicy already exists in servicePolicies + boolean policyExists = combinedPolicies.stream() + .anyMatch(policy -> policy.getId() != null && policy.getId().equals(rangerPolicy.getId())); + + // Add or remove rangerPolicy based on action + if (action.equalsIgnoreCase(RangerConstants.ACTION_DELETE)) { + // Remove rangerPolicy if it exists + if (policyExists) { + combinedPolicies.removeIf(policy -> policy.getId() != null && + policy.getId().equals(rangerPolicy.getId())); + LOG.info("Removed policy {} from combinedPolicies", rangerPolicy.getId()); + } else { + LOG.error("Failed to delete policy {} from combinedPolicies", rangerPolicy.getId()); + } + } else { + // CREATE or UPDATE: Remove old version and add new version + combinedPolicies.removeIf(policy -> policy.getId() != null && + policy.getId().equals(rangerPolicy.getId())); + combinedPolicies.add(rangerPolicy); + LOG.info("Added/Updated policy {} with users: {}", + rangerPolicy.getId(), rangerPolicy.getPolicyItems()); + } + + return combinedPolicies; + } + + /* + * Case 1: Create - default bucket with * or new bucket path + * Case 2: Update - default bucket with * or existing bucket path + * Case 3: Delete - default bucket with * (this goes for complete delete) or + * existing bucket path (this goes an update bucketPolicy) + * Case 4: No Change - existing default bucket with * or with path but not in + * affected path + */ + + List populateBucketMap(Map>> bucketMap, + List combinedPolicies, + String bucketName, RangerPolicy affectedPolicy) { + List affectedPolicies = new ArrayList<>(); + if (LOG.isDebugEnabled()) { + LOG.debug(" ======> ServiceDBStore.populateBucketMap()"); + } + if (CollectionUtils.isNotEmpty(combinedPolicies)) { + List affectedPaths = affectedPolicy.getResources().values().stream() + .flatMap(resource -> resource.getValues().stream()) + .collect(Collectors.toList()); + + for (String affectedPath : affectedPaths) { + String bucketPart = affectedPath.split("/", 2)[0]; + for (RangerPolicy rangerPolicy : combinedPolicies) { + for (RangerPolicyResource resource : rangerPolicy.getResources().values()) { + for (String s3Path : resource.getValues()) { + String prefixBucketPath = s3Path.split("/", 2)[0]; + if (affectedPath.startsWith("*") + || (affectedPaths.contains("*") && prefixBucketPath.equals(bucketName))) { + addToBucketMap(bucketMap, bucketName, rangerPolicy, s3Path); + affectedPolicies.add(rangerPolicy); + break; + } else if (prefixBucketPath.equals(bucketPart)) { + addToBucketMap(bucketMap, bucketPart, rangerPolicy, s3Path); + affectedPolicies.add(rangerPolicy); + } + } + } + } + } + } + return affectedPolicies; + } + + // Add to bucket map helper method + private void addToBucketMap(Map>> bucketMap, String bucketPart, + RangerPolicy policy, String path) { + bucketMap.putIfAbsent(bucketPart, new HashMap<>()); + bucketMap.get(bucketPart).putIfAbsent(policy, new HashSet<>()); + bucketMap.get(bucketPart).get(policy).add(path); + } - public enum VERSION_TYPE { POLICY_VERSION, TAG_VERSION, ROLE_VERSION, GDS_VERSION } + // Process bucket policies + void processPolicies(Map>> bucketMap, S3Client s3, IamClient iamClient, + Set snapshotArns) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug(" ======> ServiceDBStore.processPolicies()"); + } + // Optimization: Create IAM identity cache to avoid redundant IAM API lookups + // Cache key format: "entityType:entityName" -> "entityArn" + // Example: "user:john" -> "arn:aws:iam::123456789012:user/john" + Map iamArnCache = new HashMap<>(); + + for (Entry>> entry : bucketMap.entrySet()) { + String bucketName = entry.getKey(); + List statements = new ArrayList<>(); + // Start with snapshot ARNs (old resource paths) so that stale IAM + // statements are removed even when the resource path has changed. + Set rangerManagedResources = new HashSet<>(snapshotArns); + + for (Entry> policyEntry : entry.getValue().entrySet()) { + List s3Resources = policyEntry.getValue().stream() + .map(s3path -> RangerS3Constants.S3_RESOURCE_PATH_ARN + s3path) + .collect(Collectors.toList()); + rangerManagedResources.addAll(s3Resources); + statements.addAll(createBucketPolicyStatement(policyEntry.getKey().getPolicyItems(), + RangerS3Constants.ALLOW, s3Resources, iamClient, iamArnCache)); + statements.addAll(createBucketPolicyStatement(policyEntry.getKey().getDenyPolicyItems(), + RangerS3Constants.DENY, s3Resources, iamClient, iamArnCache)); + } + + List mergedStatements = mergeWithIAMStatements(s3, bucketName, statements, + rangerManagedResources); + String policyJson = mapToS3BucketPolicyObject(mergedStatements); + if (StringUtils.isNotEmpty(policyJson)) { + updateBucketPolicyIfChanged(s3, entry.getKey(), policyJson); + } + } + } + + // Update bucket policy if it has changed + private void updateBucketPolicyIfChanged(S3Client s3, String bucketName, String newPolicyJson) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug(" ======> ServiceDBStore.updateBucketPolicyIfChanged()"); + } + ObjectMapper objectMapper = new ObjectMapper(); + Object newPolicy = objectMapper.readTree(newPolicyJson); + String currentPolicyJson = getBucketPolicy(s3, bucketName); + boolean shouldUpdate = false; + if (StringUtils.isEmpty(currentPolicyJson)) { + shouldUpdate = true; + LOG.info("No existing policy in IAM for bucket: {}. Creating new policy.", bucketName); + } else { + Object currentPolicy = objectMapper.readTree(currentPolicyJson); + shouldUpdate = !newPolicy.equals(currentPolicy); + + if (!shouldUpdate) { + LOG.info("Policy unchanged for bucket: {}. Skipping update.", bucketName); + } + } + + if (shouldUpdate) { + if (LOG.isDebugEnabled()) { + LOG.debug("Updating bucket policy in IAM for: {}", bucketName); + } + putBucketPolicy(s3, bucketName, newPolicyJson); + LOG.info("Successfully updated bucket policy for: {}", bucketName); + } + } + + private List createBucketPolicyStatement(List policyItems, String effect, + List s3Resources, IamClient iamClient, Map iamArnCache) { + List statements = new ArrayList<>(); + if (LOG.isDebugEnabled()) { + LOG.debug(" ======> ServiceDBStore.createBucketPolicyStatement()"); + } + if (CollectionUtils.isNotEmpty(policyItems) && CollectionUtils.isNotEmpty(s3Resources)) { + for (String s3Resource : s3Resources) { + for (RangerPolicyItem policyItem : policyItems) { + List accesses = policyItem.getAccesses(); + List accessTypes = new ArrayList<>(); + + for (RangerPolicyItemAccess access : accesses) { + accessTypes.add(access.getType()); + } + PolicyStatement statement = prepareBucketPolicyStatement(s3Resource, accessTypes, effect, + policyItem.getUsers(), policyItem.getGroups(), policyItem.getRoles(), iamClient, + iamArnCache); + statements.add(statement); + } + } + return statements; + } + return statements; + } + + private PolicyStatement prepareBucketPolicyStatement(String s3Resources, List accessTypes, String effect, + List users, List groups, List roles, IamClient iamClient, + Map iamArnCache) { + PolicyStatement statement = new PolicyStatement(); + statement.setEffect(effect); + + // Set multiple AWS account IDs in the Principal + Map> principal = new HashMap<>(); + List awsAccounts = new ArrayList<>(); + awsAccounts = addAccounts(awsAccounts, users, "user", iamClient, iamArnCache); + awsAccounts = addAccounts(awsAccounts, groups, "group", iamClient, iamArnCache); + awsAccounts = addAccounts(awsAccounts, roles, "role", iamClient, iamArnCache); + + principal.put(RangerS3Constants.AWS, awsAccounts); + statement.setPrincipal(principal); + statement.setAction(accessTypes); + statement.setResource(s3Resources); + return statement; + } + + private List addAccounts(List awsAccounts, List entities, String entityType, + IamClient iamClient, Map iamArnCache) { + if (CollectionUtils.isNotEmpty(entities)) { + if ("group".equalsIgnoreCase(entityType)) { + LOG.warn( + "Group entityType is not supported for S3 bucket policies; skipping groups from principal list"); + return awsAccounts; + } + try { + // Define a map to associate entity types with the respective IAM call + Map> entityArnExtractor = new HashMap<>(); + entityArnExtractor.put("user", + entity -> iamClient.getUser(GetUserRequest.builder().build()).user().arn()); + entityArnExtractor.put("role", + entity -> iamClient.getRole(GetRoleRequest.builder().roleName(entity).build()).role().arn()); + + for (String entity : entities) { + try { + // Optimization: Check cache first to avoid redundant IAM API calls + String cacheKey = entityType + ":" + entity; + String entityArn = iamArnCache.get(cacheKey); + + if (entityArn == null) { + // Cache miss - fetch from IAM + entityArn = entityArnExtractor.get(entityType).apply(entity); + iamArnCache.put(cacheKey, entityArn); + if (LOG.isDebugEnabled()) { + LOG.debug("Cached IAM identity for {}: {}", cacheKey, entityArn); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Using cached IAM identity for {}", cacheKey); + } + } + + // Parse ARN to extract account ID + // Expected ARN format: arn:aws:iam::123456789012:user/username or + // arn:aws:iam::123456789012:role/rolename + String[] arnParts = entityArn.split(":"); + if (arnParts.length < 5) { + throw new IllegalArgumentException("Invalid ARN format: " + entityArn); + } + String accountId = arnParts[4]; // Account ID is the 5th segment of the ARN + awsAccounts.add( + RangerS3Constants.S3_AWS_ACCOUNT_URN + accountId + ":" + entityType + "/" + entity); + } catch (Exception e) { + LOG.error("Failed to retrieve account ID for entity '{}': {}", entity, e.getMessage()); + throw e; + } + } + } catch (SdkServiceException e) { + LOG.error("IAM service error: {}", e.getMessage()); + throw e; + } catch (Exception e) { + LOG.error("General Exception to retrieve and addAccounts: {}", e.getMessage()); + throw e; + } + } + return awsAccounts; + } + + /** + * Merge Ranger policy statements with existing IAM statements + * + * @param s3 S3 client + * @param bucketName Name of the S3 bucket + * @param rangerStatements Policy statements created from Ranger DB policies + * @return Merged list of statements (IAM existing + Ranger) + * @throws Exception if merge fails + */ + List mergeWithIAMStatements(S3Client s3, String bucketName, + List rangerStatements, + Set rangerManagedResources) throws Exception { + ObjectMapper objectMapper = new ObjectMapper(); + List mergedStatements = new ArrayList<>(); + + // Step 1: Get existing IAM policy JSON + String existingIAMPolicyJson = getBucketPolicy(s3, bucketName); + + if (StringUtils.isNotEmpty(existingIAMPolicyJson)) { + try { + // Step 2: Parse IAM JSON to extract statements + S3BucketPolicy existingIAMPolicy = objectMapper.readValue( + existingIAMPolicyJson, S3BucketPolicy.class); + + if (existingIAMPolicy.getStatement() != null) { + List iamStatements = existingIAMPolicy.getStatement(); + + LOG.info("Found {} existing statements in IAM for bucket: {}", + iamStatements.size(), bucketName); + + // Step 3: Identify IAM statements that are NOT managed by Ranger + List iamOnlyStatements = extractIAMOnlyStatements( + iamStatements, rangerManagedResources); + + LOG.info("Preserving {} IAM-only statements for bucket: {}", + iamOnlyStatements.size(), bucketName); + + // Step 4: Add IAM-only statements first + mergedStatements.addAll(iamOnlyStatements); + } + } catch (Exception e) { + LOG.error("Failed to parse existing IAM policy for bucket: {}. " + + "Proceeding with Ranger statements only.", bucketName, e); + } + } else { + LOG.info("No existing IAM policy found for bucket: {}. " + + "Creating new policy with Ranger statements.", bucketName); + } + + // Step 5: Add all Ranger statements + mergedStatements.addAll(rangerStatements); + + LOG.info("Merged policy for bucket: {} contains {} total statements " + + "({} IAM-only + {} Ranger-managed)", + bucketName, mergedStatements.size(), + mergedStatements.size() - rangerStatements.size(), + rangerStatements.size()); + + return mergedStatements; + } + + /** + * Extract statements from IAM that are NOT managed by Ranger. + * A statement is considered Ranger-managed if its resource ARN appears in + * {@code rangerManagedResources}, which is the union of the snapshot ARNs + * (old resource paths before this update) and the current Ranger ARNs + * (new resource paths after this update). This ensures stale statements are + * removed even when the resource path itself has changed. * + * + * @param iamStatements Statements from the existing IAM/S3 bucket + * policy + * @param rangerManagedResources Union of old (snapshot) and new Ranger resource + * ARNs + * @return List of statements that are NOT managed by Ranger (should be + * preserved) + */ + List extractIAMOnlyStatements( + List iamStatements, + Set rangerManagedResources) { + if (LOG.isDebugEnabled()) { + LOG.debug(" ======> ServiceDBStore.extractIAMOnlyStatements()"); + } + + List iamOnlyStatements = new ArrayList<>(); + + if (CollectionUtils.isEmpty(iamStatements)) { + return iamOnlyStatements; + } + + for (PolicyStatement iamStmt : iamStatements) { + // If this statement's resource is in the managed set (old or new ARN), + // it is a stale Ranger statement — drop it and let new statements replace it. + if (rangerManagedResources.contains(iamStmt.getResource())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Dropping stale Ranger-managed statement for resource: {}", + iamStmt.getResource()); + } + continue; + } + + // Resource is not managed by Ranger — preserve it as an IAM-only statement + iamOnlyStatements.add(iamStmt); + if (LOG.isDebugEnabled()) { + LOG.debug("Preserving IAM-only statement: {}", iamStmt.getResource()); + } + } + return iamOnlyStatements; + } + + /** + * Compare two policy statements for equivalence + * Used to determine if an IAM statement matches a Ranger statement + * + * @param stmt1 First statement + * @param stmt2 Second statement + * @return true if statements are functionally equivalent + */ + boolean statementsMatch(PolicyStatement stmt1, PolicyStatement stmt2) { + if (stmt1 == null || stmt2 == null) { + return false; + } + + // Compare effect (Allow/Deny) + if (!StringUtils.equals(stmt1.getEffect(), stmt2.getEffect())) { + return false; + } + + // Compare resource path + if (!StringUtils.equals(stmt1.getResource(), stmt2.getResource())) { + return false; + } + + // Compare actions (order doesn't matter) + if (!compareActions(stmt1.getAction(), stmt2.getAction())) { + return false; + } + + // Compare principals (order doesn't matter) + if (!comparePrincipals(stmt1.getPrincipal(), stmt2.getPrincipal())) { + return false; + } + + return true; + } + + /** + * Compare action lists (order-independent) + */ + private boolean compareActions(List actions1, List actions2) { + if (actions1 == null && actions2 == null) { + return true; + } + if (actions1 == null || actions2 == null) { + return false; + } + + Set set1 = new HashSet<>(actions1); + Set set2 = new HashSet<>(actions2); + + return set1.equals(set2); + } + + /** + * Compare principal maps (order-independent) + */ + private boolean comparePrincipals(Map> principal1, + Map> principal2) { + if (principal1 == null && principal2 == null) { + return true; + } + if (principal1 == null || principal2 == null) { + return false; + } + + // Get AWS principals + List awsPrincipals1 = principal1.get(RangerS3Constants.AWS); + List awsPrincipals2 = principal2.get(RangerS3Constants.AWS); + + if (awsPrincipals1 == null && awsPrincipals2 == null) { + return true; + } + if (awsPrincipals1 == null || awsPrincipals2 == null) { + return false; + } + + Set set1 = new HashSet<>(awsPrincipals1); + Set set2 = new HashSet<>(awsPrincipals2); + + return set1.equals(set2); + } + + private String mapToS3BucketPolicyObject(List statements) { + S3BucketPolicy s3BucketPolicy = new S3BucketPolicy(); + s3BucketPolicy.setVersion(RangerS3Constants.S3_POLICY_LANGUAGE_VERSION); + s3BucketPolicy.setStatement(statements); + + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + + try { + return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(s3BucketPolicy); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public void putBucketPolicy(S3Client s3, String bucketName, String policy) { + if (LOG.isDebugEnabled()) { + LOG.debug(" ======> ServiceDBStore.putBucketPolicy()"); + } + try { + PutBucketPolicyRequest putBucketPolicyRequest = PutBucketPolicyRequest.builder() + .bucket(bucketName) + .policy(policy) + .build(); + + s3.putBucketPolicy(putBucketPolicyRequest); + } catch (S3Exception e) { + LOG.error(e.awsErrorDetails().toString()); + throw restErrorUtil.createRESTException(e.awsErrorDetails().toString()); + } + } + + public void deleteBucketPolicy(S3Client s3, String bucketName) { + if (LOG.isDebugEnabled()) { + LOG.debug(" ======> ServiceDBStore.deleteBucketPolicy()"); + } + try { + DeleteBucketPolicyRequest deleteBucketPolicyRequest = DeleteBucketPolicyRequest.builder() + .bucket(bucketName) + .build(); + + s3.deleteBucketPolicy(deleteBucketPolicyRequest); + } catch (S3Exception e) { + LOG.error(e.awsErrorDetails().toString()); + throw restErrorUtil.createRESTException(e.awsErrorDetails().toString()); + } + } + + public String getBucketPolicy(S3Client s3, String bucketName) { + String policyText; + GetBucketPolicyRequest policyReq = GetBucketPolicyRequest.builder() + .bucket(bucketName) + .build(); + + try { + GetBucketPolicyResponse policyRes = s3.getBucketPolicy(policyReq); + policyText = policyRes.policy(); + return policyText; + } catch (S3Exception e) { + if (e.awsErrorDetails().errorCode().equals("NoSuchBucketPolicy")) { + LOG.error("Bucket: " + bucketName + " does not have a policy."); + return ""; + } else { + LOG.error("Failed to get policy for bucket " + bucketName + ": " + e.getMessage()); + throw restErrorUtil.createRESTException(e.awsErrorDetails().toString()); + } + } + } + + public enum JSON_FILE_NAME_TYPE { + POLICY, ROLE + } + + public enum VERSION_TYPE { + POLICY_VERSION, TAG_VERSION, ROLE_VERSION, GDS_VERSION + } public enum METRIC_TYPE { USER_GROUP { @@ -6665,7 +8065,9 @@ public static METRIC_TYPE getMetricTypeByName(final String metricTypeName) { abstract String getMetric(ServiceDBStore ref, SearchCriteria searchCriteria); } - public enum REMOVE_REF_TYPE { USER, GROUP, ROLE } + public enum REMOVE_REF_TYPE { + USER, GROUP, ROLE + } private static class RangerPolicyDeltaComparator implements Comparator, java.io.Serializable { @Override @@ -6675,44 +8077,47 @@ public int compare(RangerPolicyDelta me, RangerPolicyDelta other) { } public static class ServiceVersionUpdater implements Runnable { - final Long serviceId; - final RangerDaoManager daoManager; - final VERSION_TYPE versionType; - final String zoneName; - final Integer policyDeltaChange; - final RangerPolicy policy; + final Long serviceId; + final RangerDaoManager daoManager; + final VERSION_TYPE versionType; + final String zoneName; + final Integer policyDeltaChange; + final RangerPolicy policy; final ServiceTags.TagsChangeType tagChangeType; - final Long resourceId; - final Long tagId; + final Long resourceId; + final Long tagId; long version = -1; - public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType, Integer policyDeltaType) { + public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType, + Integer policyDeltaType) { this(daoManager, serviceId, versionType, null, policyDeltaType, null); } - public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType, String zoneName, Integer policyDeltaType, RangerPolicy policy) { - this.serviceId = serviceId; - this.daoManager = daoManager; - this.versionType = versionType; + public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType, + String zoneName, Integer policyDeltaType, RangerPolicy policy) { + this.serviceId = serviceId; + this.daoManager = daoManager; + this.versionType = versionType; this.policyDeltaChange = policyDeltaType; - this.zoneName = zoneName; - this.policy = policy; - this.tagChangeType = ServiceTags.TagsChangeType.NONE; - this.resourceId = null; - this.tagId = null; - } - - public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType, ServiceTags.TagsChangeType tagChangeType, Long resourceId, Long tagId) { - this.serviceId = serviceId; - this.daoManager = daoManager; - this.versionType = versionType; - this.zoneName = null; + this.zoneName = zoneName; + this.policy = policy; + this.tagChangeType = ServiceTags.TagsChangeType.NONE; + this.resourceId = null; + this.tagId = null; + } + + public ServiceVersionUpdater(RangerDaoManager daoManager, Long serviceId, VERSION_TYPE versionType, + ServiceTags.TagsChangeType tagChangeType, Long resourceId, Long tagId) { + this.serviceId = serviceId; + this.daoManager = daoManager; + this.versionType = versionType; + this.zoneName = null; this.policyDeltaChange = null; - this.policy = null; - this.tagChangeType = tagChangeType; - this.resourceId = resourceId; - this.tagId = tagId; + this.policy = null; + this.tagChangeType = tagChangeType; + this.resourceId = resourceId; + this.tagId = tagId; } @Override @@ -6737,12 +8142,12 @@ public String toString() { } private class AssociatePolicyLabel implements Runnable { - private final String policyLabel; + private final String policyLabel; private final XXPolicy xPolicy; AssociatePolicyLabel(String policyLabel, XXPolicy xPolicy) { this.policyLabel = policyLabel; - this.xPolicy = xPolicy; + this.xPolicy = xPolicy; } @Override @@ -6772,10 +8177,13 @@ private void getOrCreateLabel() { xxPolicyLabel = daoMgr.getXXPolicyLabels().create(xxPolicyLabel); } - // doing a find to check if the label is already associated with the policy (may happen in concurrent sessions) - List xxPolicyLabelMapList = daoMgr.getXXPolicyLabelMap().findByPolicyIdAndLabelId(xPolicy.getId(), xxPolicyLabel.getId()); + // doing a find to check if the label is already associated with the policy (may + // happen in concurrent sessions) + List xxPolicyLabelMapList = daoMgr.getXXPolicyLabelMap() + .findByPolicyIdAndLabelId(xPolicy.getId(), xxPolicyLabel.getId()); if (xxPolicyLabelMapList != null && !xxPolicyLabelMapList.isEmpty()) { - LOG.info("Policy with id {} already linked to label with id = {}", xPolicy.getId(), xxPolicyLabel.getId()); + LOG.info("Policy with id {} already linked to label with id = {}", xPolicy.getId(), + xxPolicyLabel.getId()); } else { XXPolicyLabelMap xxPolicyLabelMap = new XXPolicyLabelMap(); @@ -6784,7 +8192,8 @@ private void getOrCreateLabel() { xxPolicyLabelMap = rangerAuditFields.populateAuditFieldsForCreate(xxPolicyLabelMap); - LOG.debug("Creating a link for policy Id = {} to labelId = {}", xPolicy.getId(), xxPolicyLabel.getId()); + LOG.debug("Creating a link for policy Id = {} to labelId = {}", xPolicy.getId(), + xxPolicyLabel.getId()); daoMgr.getXXPolicyLabelMap().create(xxPolicyLabelMap); } } else { diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerConstants.java b/security-admin/src/main/java/org/apache/ranger/common/RangerConstants.java index 3a8e33ee9f2..250fd30d91e 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerConstants.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerConstants.java @@ -49,6 +49,8 @@ public class RangerConstants extends RangerCommonEnums { // Action constants public static final String ACTION_EDIT = "edit"; public static final String ACTION_CHANGE = "change"; + public static final String ACTION_CREATE = "create"; + public static final String ACTION_UPDATE = "update"; public static final String ACTION_DELETE = "delete"; public static final String ACTION_MARK_SPAM = "mark_spam"; public static final String ACTION_RATE = "rate"; diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 25534c214be..6813efb5315 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -106,6 +106,7 @@ import org.apache.ranger.service.RangerServiceDefService; import org.apache.ranger.service.RangerServiceService; import org.apache.ranger.service.XUserService; +import org.apache.ranger.services.s3.RangerS3Constants; import org.apache.ranger.view.RangerExportPolicyList; import org.apache.ranger.view.RangerPluginInfoList; import org.apache.ranger.view.RangerPolicyList; @@ -1636,11 +1637,26 @@ public RangerPolicy createPolicy(RangerPolicy policy, @Context HttpServletReques LOG.debug("<== ServiceREST.createPolicy({}): {}", policy, ret); + if (StringUtils.isBlank(policy.getServiceType())) { + LOG.error("Policy {} missing serviceType; cannot determine service behavior", policy.getId()); + throw new IllegalStateException("Policy serviceType is missing"); + } + if (StringUtils.equalsIgnoreCase(policy.getServiceType(), RangerS3Constants.S3)) { + svcStore.createS3BucketPolicy(policy, RangerConstants.ACTION_CREATE, null); + } + return ret; } if (ret == null) { ret = createPolicyUnconditionally(policy); + if (StringUtils.isBlank(policy.getServiceType())) { + LOG.error("Policy {} missing serviceType; cannot determine service behavior", policy.getId()); + throw new IllegalStateException("Policy serviceType is missing"); + } + if (StringUtils.equalsIgnoreCase(policy.getServiceType(), RangerS3Constants.S3)) { + svcStore.createS3BucketPolicy(policy, RangerConstants.ACTION_CREATE, null); + } } } catch (WebApplicationException excp) { throw excp; @@ -1784,7 +1800,16 @@ public RangerPolicy updatePolicy(RangerPolicy policy, @PathParam("id") Long id) bizUtil.blockAuditorRoleUser(); + RangerPolicy oldPolicy = StringUtils.equalsIgnoreCase(RangerS3Constants.S3, policy.getServiceType()) + ? svcStore.getPolicy(policy.getId()) : null; ret = svcStore.updatePolicy(policy); + if (StringUtils.isBlank(policy.getServiceType())) { + LOG.error("Policy {} missing serviceType; cannot determine service behavior", policy.getId()); + throw new IllegalStateException("Policy serviceType is missing"); + } + if (StringUtils.equalsIgnoreCase(RangerS3Constants.S3, policy.getServiceType())) { + svcStore.createS3BucketPolicy(policy, RangerConstants.ACTION_UPDATE, oldPolicy); + } } catch (WebApplicationException excp) { throw excp; } catch (Throwable excp) { @@ -1823,6 +1848,13 @@ public void deletePolicy(@PathParam("id") Long id) { bizUtil.blockAuditorRoleUser(); svcStore.deletePolicy(policy); + if (StringUtils.isBlank(policy.getServiceType())) { + LOG.error("Policy {} missing serviceType; cannot determine service behavior", policy.getId()); + throw new IllegalStateException("Policy serviceType is missing"); + } + if (StringUtils.equalsIgnoreCase(policy.getServiceType(), RangerS3Constants.S3)) { + // policy is read above (before deletion) so it still holds the original resources + svcStore.createS3BucketPolicy(policy, RangerConstants.ACTION_DELETE, policy); } } catch (WebApplicationException excp) { throw excp; } catch (Throwable excp) { diff --git a/security-admin/src/main/java/org/apache/ranger/s3/PolicyStatement.java b/security-admin/src/main/java/org/apache/ranger/s3/PolicyStatement.java new file mode 100644 index 00000000000..75d49710453 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/s3/PolicyStatement.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.s3; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class PolicyStatement { + @JsonProperty("Effect") + private String effect; + @JsonProperty("Principal") + private Map> principal; + @JsonProperty("Action") + private List action; + @JsonProperty("Resource") + private String resource; + + // Getters and setters + public String getEffect() { + return effect; + } + + public void setEffect(String effect) { + this.effect = effect; + } + + public Map> getPrincipal() { + return principal; + } + + public void setPrincipal(Map> principal) { + this.principal = principal; + } + + public List getAction() { + return action; + } + + @JsonProperty("Action") + public void setAction(Object action) { + if (action == null) { + this.action = new ArrayList<>(); + } else if (action instanceof String) { + // Handle single action as string + this.action = new ArrayList<>(); + this.action.add((String) action); + } else if (action instanceof List) { + // Handle multiple actions as list + this.action = (List) action; + } else { + // Fallback + this.action = new ArrayList<>(); + } + } + + public void setAction(List action) { + this.action = action; + } + + public String getResource() { + return resource; + } + + public void setResource(String resource) { + this.resource = resource; + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/s3/S3BucketPolicy.java b/security-admin/src/main/java/org/apache/ranger/s3/S3BucketPolicy.java new file mode 100644 index 00000000000..d1aac4e2c12 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/s3/S3BucketPolicy.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.s3; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class S3BucketPolicy { + @JsonProperty("Version") + private String version; + @JsonProperty("Statement") + private List statement; + + // Getters and Setters + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public List getStatement() { + return statement; + } + + public void setStatement(List statement) { + this.statement = statement; + } +} diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml index 164b63d027a..a0384263dfb 100644 --- a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml +++ b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml @@ -485,7 +485,7 @@ ranger.resource.lookup.timeout.value.in.ms - 1000 + 10000 diff --git a/security-admin/src/main/webapp/react-webapp/src/images/serviceDefIcons/s3/icon.svg b/security-admin/src/main/webapp/react-webapp/src/images/serviceDefIcons/s3/icon.svg new file mode 100644 index 00000000000..425fe62783d --- /dev/null +++ b/security-admin/src/main/webapp/react-webapp/src/images/serviceDefIcons/s3/icon.svg @@ -0,0 +1,18 @@ + + + Icon-Architecture/64/Arch_Amazon-Simple-Storage-Service_64 + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java index 5a36e7f297e..16d7b79c300 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java @@ -2920,6 +2920,672 @@ public void test53GetPolicyByGUIDAndZoneName() throws Exception { Mockito.verify(policyService).getPopulatedViewObject(xPolicy); } + // ==================== S3 Bucket Policy Tests ==================== + + /** + * Helper method to create an S3 policy for testing + */ + private RangerPolicy createS3Policy(Long id, String policyName, String bucketPath, + List users, List accessTypes) { + RangerPolicy policy = new RangerPolicy(); + policy.setId(id); + policy.setName(policyName); + policy.setService("s3-service"); + policy.setIsEnabled(true); + + // Create policy items with users and access types + List policyItems = new ArrayList<>(); + RangerPolicyItem policyItem = new RangerPolicyItem(); + policyItem.setUsers(users); + + List accesses = new ArrayList<>(); + for (String accessType : accessTypes) { + RangerPolicyItemAccess access = new RangerPolicyItemAccess(); + access.setType(accessType); + access.setIsAllowed(true); + accesses.add(access); + } + policyItem.setAccesses(accesses); + policyItems.add(policyItem); + policy.setPolicyItems(policyItems); + + // Create resources with bucket path + Map resources = new HashMap<>(); + RangerPolicyResource pathResource = new RangerPolicyResource(); + pathResource.setValues(java.util.Collections.singletonList(bucketPath)); + resources.put("path", pathResource); + policy.setResources(resources); + + return policy; + } + + @Test + public void testCombinePolicies_CreateNewPolicy() { + // Setup: empty service policies, new policy to add + List servicePolicies = new ArrayList<>(); + RangerPolicy newPolicy = createS3Policy(1L, "test-policy", "test-bucket/*", + java.util.Arrays.asList("user1"), + java.util.Arrays.asList("s3:GetObject")); + + // Execute: CREATE action + List result = serviceDBStore.combinePolicies(servicePolicies, newPolicy, "CREATE"); + + // Assert: new policy should be added + Assert.assertEquals(1, result.size()); + Assert.assertEquals(newPolicy.getId(), result.get(0).getId()); + } + + @Test + public void testCombinePolicies_UpdateExistingPolicy() { + // Setup: existing policy in service policies + RangerPolicy existingPolicy = createS3Policy(1L, "test-policy", "test-bucket/*", + java.util.Arrays.asList("user1"), + java.util.Arrays.asList("s3:GetObject")); + List servicePolicies = new ArrayList<>(); + servicePolicies.add(existingPolicy); + + // Updated policy with same ID but different users + RangerPolicy updatedPolicy = createS3Policy(1L, "test-policy", "test-bucket/*", + java.util.Arrays.asList("user1", "user2"), + java.util.Arrays.asList("s3:GetObject", "s3:PutObject")); + + // Execute: UPDATE action + List result = serviceDBStore.combinePolicies(servicePolicies, updatedPolicy, "UPDATE"); + + // Assert: policy should be updated (old removed, new added) + Assert.assertEquals(1, result.size()); + Assert.assertEquals(updatedPolicy.getId(), result.get(0).getId()); + Assert.assertEquals(2, result.get(0).getPolicyItems().get(0).getUsers().size()); + } + + @Test + public void testCombinePolicies_DeletePolicy() { + // Setup: existing policies in service + RangerPolicy policy1 = createS3Policy(1L, "policy-1", "bucket1/*", + java.util.Arrays.asList("user1"), + java.util.Arrays.asList("s3:GetObject")); + RangerPolicy policy2 = createS3Policy(2L, "policy-2", "bucket2/*", + java.util.Arrays.asList("user2"), + java.util.Arrays.asList("s3:GetObject")); + List servicePolicies = new ArrayList<>(); + servicePolicies.add(policy1); + servicePolicies.add(policy2); + + // Execute: DELETE policy1 + List result = serviceDBStore.combinePolicies(servicePolicies, policy1, "DELETE"); + + // Assert: policy1 should be removed, only policy2 remains + Assert.assertEquals(1, result.size()); + Assert.assertEquals(policy2.getId(), result.get(0).getId()); + } + + @Test + public void testCombinePolicies_DeleteFromEmptyList() { + // Setup: empty service policies + List servicePolicies = new ArrayList<>(); + RangerPolicy policyToDelete = createS3Policy(1L, "test-policy", "test-bucket/*", + java.util.Arrays.asList("user1"), + java.util.Arrays.asList("s3:GetObject")); + + // Execute: DELETE action on empty list + List result = serviceDBStore.combinePolicies(servicePolicies, policyToDelete, "DELETE"); + + // Assert: result should be empty + Assert.assertEquals(0, result.size()); + } + + @Test + public void testCombinePolicies_AddMultiplePolicies() { + // Setup: one existing policy + RangerPolicy existingPolicy = createS3Policy(1L, "policy-1", "bucket1/*", + java.util.Arrays.asList("user1"), + java.util.Arrays.asList("s3:GetObject")); + List servicePolicies = new ArrayList<>(); + servicePolicies.add(existingPolicy); + + // Execute: CREATE new policy + RangerPolicy newPolicy = createS3Policy(2L, "policy-2", "bucket2/*", + java.util.Arrays.asList("user2"), + java.util.Arrays.asList("s3:PutObject")); + List result = serviceDBStore.combinePolicies(servicePolicies, newPolicy, "CREATE"); + + // Assert: both policies should be present + Assert.assertEquals(2, result.size()); + } + + @Test + public void testPopulateBucketMap_SingleBucketSinglePolicy() { + // Setup: policy with single bucket path + RangerPolicy policy = createS3Policy(1L, "test-policy", "test-bucket/data/*", + java.util.Arrays.asList("user1"), + java.util.Arrays.asList("s3:GetObject")); + List combinedPolicies = java.util.Arrays.asList(policy); + Map>> bucketMap = new HashMap<>(); + + // Execute + List affectedPolicies = serviceDBStore.populateBucketMap( + bucketMap, combinedPolicies, "test-bucket", policy); + + // Assert: bucket map should contain the policy + Assert.assertTrue(bucketMap.containsKey("test-bucket")); + Assert.assertEquals(1, affectedPolicies.size()); + Assert.assertEquals(policy.getId(), affectedPolicies.get(0).getId()); + } + + @Test + public void testPopulateBucketMap_MultipleBuckets() { + // Setup: policies with different buckets + RangerPolicy policy1 = createS3Policy(1L, "policy-1", "bucket1/data/*", + java.util.Arrays.asList("user1"), + java.util.Arrays.asList("s3:GetObject")); + RangerPolicy policy2 = createS3Policy(2L, "policy-2", "bucket2/logs/*", + java.util.Arrays.asList("user2"), + java.util.Arrays.asList("s3:GetObject")); + List combinedPolicies = java.util.Arrays.asList(policy1, policy2); + Map>> bucketMap = new HashMap<>(); + + // Execute: affected policy is policy1 + List affectedPolicies = serviceDBStore.populateBucketMap( + bucketMap, combinedPolicies, "bucket1", policy1); + + // Assert: only bucket1 should be in map + Assert.assertTrue(bucketMap.containsKey("bucket1")); + Assert.assertFalse(bucketMap.containsKey("bucket2")); + Assert.assertEquals(1, affectedPolicies.size()); + } + + @Test + public void testPopulateBucketMap_WildcardPath() { + // Setup: policy with wildcard path + RangerPolicy wildcardPolicy = createS3Policy(1L, "wildcard-policy", "*", + java.util.Arrays.asList("user1"), + java.util.Arrays.asList("s3:*")); + List combinedPolicies = java.util.Arrays.asList(wildcardPolicy); + Map>> bucketMap = new HashMap<>(); + + // Execute + List affectedPolicies = serviceDBStore.populateBucketMap( + bucketMap, combinedPolicies, "default-bucket", wildcardPolicy); + + // Assert: default bucket should be in map + Assert.assertTrue(bucketMap.containsKey("default-bucket")); + Assert.assertEquals(1, affectedPolicies.size()); + } + + @Test + public void testPopulateBucketMap_EmptyPolicies() { + // Setup: empty policies list + List combinedPolicies = new ArrayList<>(); + Map>> bucketMap = new HashMap<>(); + RangerPolicy affectedPolicy = createS3Policy(1L, "test-policy", "test-bucket/*", + java.util.Arrays.asList("user1"), + java.util.Arrays.asList("s3:GetObject")); + + // Execute + List affectedPolicies = serviceDBStore.populateBucketMap( + bucketMap, combinedPolicies, "test-bucket", affectedPolicy); + + // Assert: bucket map should be empty + Assert.assertTrue(bucketMap.isEmpty()); + Assert.assertTrue(affectedPolicies.isEmpty()); + } + + @Test + public void testStatementsMatch_IdenticalStatements() { + // Setup: two identical statements + org.apache.ranger.s3.PolicyStatement stmt1 = new org.apache.ranger.s3.PolicyStatement(); + stmt1.setEffect("Allow"); + stmt1.setResource("arn:aws:s3:::test-bucket/*"); + stmt1.setAction(java.util.Arrays.asList("s3:GetObject", "s3:PutObject")); + + org.apache.ranger.s3.PolicyStatement stmt2 = new org.apache.ranger.s3.PolicyStatement(); + stmt2.setEffect("Allow"); + stmt2.setResource("arn:aws:s3:::test-bucket/*"); + stmt2.setAction(java.util.Arrays.asList("s3:GetObject", "s3:PutObject")); + + // Execute & Assert + Assert.assertTrue(serviceDBStore.statementsMatch(stmt1, stmt2)); + } + + @Test + public void testStatementsMatch_DifferentEffect() { + // Setup: statements with different effects + org.apache.ranger.s3.PolicyStatement stmt1 = new org.apache.ranger.s3.PolicyStatement(); + stmt1.setEffect("Allow"); + stmt1.setResource("arn:aws:s3:::test-bucket/*"); + stmt1.setAction(java.util.Arrays.asList("s3:GetObject")); + + org.apache.ranger.s3.PolicyStatement stmt2 = new org.apache.ranger.s3.PolicyStatement(); + stmt2.setEffect("Deny"); + stmt2.setResource("arn:aws:s3:::test-bucket/*"); + stmt2.setAction(java.util.Arrays.asList("s3:GetObject")); + + // Execute & Assert + Assert.assertFalse(serviceDBStore.statementsMatch(stmt1, stmt2)); + } + + @Test + public void testStatementsMatch_DifferentResource() { + // Setup: statements with different resources + org.apache.ranger.s3.PolicyStatement stmt1 = new org.apache.ranger.s3.PolicyStatement(); + stmt1.setEffect("Allow"); + stmt1.setResource("arn:aws:s3:::bucket1/*"); + stmt1.setAction(java.util.Arrays.asList("s3:GetObject")); + + org.apache.ranger.s3.PolicyStatement stmt2 = new org.apache.ranger.s3.PolicyStatement(); + stmt2.setEffect("Allow"); + stmt2.setResource("arn:aws:s3:::bucket2/*"); + stmt2.setAction(java.util.Arrays.asList("s3:GetObject")); + + // Execute & Assert + Assert.assertFalse(serviceDBStore.statementsMatch(stmt1, stmt2)); + } + + @Test + public void testStatementsMatch_NullStatements() { + // Setup + org.apache.ranger.s3.PolicyStatement stmt1 = new org.apache.ranger.s3.PolicyStatement(); + stmt1.setEffect("Allow"); + + // Execute & Assert + Assert.assertFalse(serviceDBStore.statementsMatch(null, stmt1)); + Assert.assertFalse(serviceDBStore.statementsMatch(stmt1, null)); + Assert.assertFalse(serviceDBStore.statementsMatch(null, null)); + } + + @Test + public void testExtractIAMOnlyStatements_NoOverlap() { + // Setup: IAM statements that don't match Ranger statements + org.apache.ranger.s3.PolicyStatement iamStmt1 = new org.apache.ranger.s3.PolicyStatement(); + iamStmt1.setEffect("Allow"); + iamStmt1.setResource("arn:aws:s3:::iam-only-bucket/*"); + iamStmt1.setAction(java.util.Arrays.asList("s3:GetObject")); + + org.apache.ranger.s3.PolicyStatement rangerStmt1 = new org.apache.ranger.s3.PolicyStatement(); + rangerStmt1.setEffect("Allow"); + rangerStmt1.setResource("arn:aws:s3:::ranger-bucket/*"); + rangerStmt1.setAction(java.util.Arrays.asList("s3:PutObject")); + + List iamStatements = java.util.Arrays.asList(iamStmt1); + List rangerStatements = java.util.Arrays.asList(rangerStmt1); + + // Execute + List result = + serviceDBStore.extractIAMOnlyStatements(iamStatements, rangerStatements); + + // Assert: IAM statement should be preserved + Assert.assertEquals(1, result.size()); + Assert.assertEquals(iamStmt1.getResource(), result.get(0).getResource()); + } + + @Test + public void testExtractIAMOnlyStatements_WithOverlap() { + // Setup: IAM and Ranger statements with one matching + org.apache.ranger.s3.PolicyStatement iamStmt1 = new org.apache.ranger.s3.PolicyStatement(); + iamStmt1.setEffect("Allow"); + iamStmt1.setResource("arn:aws:s3:::shared-bucket/*"); + iamStmt1.setAction(java.util.Arrays.asList("s3:GetObject")); + + org.apache.ranger.s3.PolicyStatement iamStmt2 = new org.apache.ranger.s3.PolicyStatement(); + iamStmt2.setEffect("Allow"); + iamStmt2.setResource("arn:aws:s3:::iam-only-bucket/*"); + iamStmt2.setAction(java.util.Arrays.asList("s3:GetObject")); + + org.apache.ranger.s3.PolicyStatement rangerStmt1 = new org.apache.ranger.s3.PolicyStatement(); + rangerStmt1.setEffect("Allow"); + rangerStmt1.setResource("arn:aws:s3:::shared-bucket/*"); + rangerStmt1.setAction(java.util.Arrays.asList("s3:GetObject")); + + List iamStatements = java.util.Arrays.asList(iamStmt1, iamStmt2); + List rangerStatements = java.util.Arrays.asList(rangerStmt1); + + // Execute + List result = + serviceDBStore.extractIAMOnlyStatements(iamStatements, rangerStatements); + + // Assert: only IAM-only statement should be preserved + Assert.assertEquals(1, result.size()); + Assert.assertEquals(iamStmt2.getResource(), result.get(0).getResource()); + } + + @Test + public void testExtractIAMOnlyStatements_EmptyIAM() { + // Setup: empty IAM statements + List iamStatements = new ArrayList<>(); + + org.apache.ranger.s3.PolicyStatement rangerStmt1 = new org.apache.ranger.s3.PolicyStatement(); + rangerStmt1.setEffect("Allow"); + rangerStmt1.setResource("arn:aws:s3:::ranger-bucket/*"); + rangerStmt1.setAction(java.util.Arrays.asList("s3:GetObject")); + List rangerStatements = java.util.Arrays.asList(rangerStmt1); + + // Execute + List result = + serviceDBStore.extractIAMOnlyStatements(iamStatements, rangerStatements); + + // Assert: result should be empty + Assert.assertEquals(0, result.size()); + } + + @Test + public void testExtractIAMOnlyStatements_AllMatch() { + // Setup: all IAM statements match Ranger statements + org.apache.ranger.s3.PolicyStatement stmt1 = new org.apache.ranger.s3.PolicyStatement(); + stmt1.setEffect("Allow"); + stmt1.setResource("arn:aws:s3:::bucket/*"); + stmt1.setAction(java.util.Arrays.asList("s3:GetObject")); + + List iamStatements = java.util.Arrays.asList(stmt1); + List rangerStatements = java.util.Arrays.asList(stmt1); + + // Execute + List result = + serviceDBStore.extractIAMOnlyStatements(iamStatements, rangerStatements); + + // Assert: no IAM-only statements + Assert.assertEquals(0, result.size()); + } + + @Test + public void testMergeWithIAMStatements_NoExistingIAMPolicy() throws Exception { + // Setup: mock S3Client with no existing policy + software.amazon.awssdk.services.s3.S3Client s3Client = Mockito.mock(software.amazon.awssdk.services.s3.S3Client.class); + Mockito.when(s3Client.getBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.GetBucketPolicyRequest.class))) + .thenThrow(software.amazon.awssdk.services.s3.model.NoSuchBucketPolicyException.class); + + org.apache.ranger.s3.PolicyStatement rangerStmt = new org.apache.ranger.s3.PolicyStatement(); + rangerStmt.setEffect("Allow"); + rangerStmt.setResource("arn:aws:s3:::test-bucket/*"); + rangerStmt.setAction(java.util.Arrays.asList("s3:GetObject")); + List rangerStatements = java.util.Arrays.asList(rangerStmt); + + // Execute + List result = + serviceDBStore.mergeWithIAMStatements(s3Client, "test-bucket", rangerStatements, java.util.Collections.emptySet()); + + // Assert: only Ranger statements should be present + Assert.assertEquals(1, result.size()); + Assert.assertEquals(rangerStmt.getResource(), result.get(0).getResource()); + } + + @Test + public void testMergeWithIAMStatements_WithExistingIAMPolicy() throws Exception { + // Setup: mock S3Client with existing IAM policy + software.amazon.awssdk.services.s3.S3Client s3Client = Mockito.mock(software.amazon.awssdk.services.s3.S3Client.class); + + // Create IAM policy JSON + String iamPolicyJson = "{\"Version\":\"2012-10-17\",\"Statement\":[" + + "{\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::iam-bucket/*\",\"Action\":[\"s3:ListBucket\"]}" + + "]}"; + + software.amazon.awssdk.services.s3.model.GetBucketPolicyResponse response = + software.amazon.awssdk.services.s3.model.GetBucketPolicyResponse.builder() + .policy(iamPolicyJson) + .build(); + + Mockito.when(s3Client.getBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.GetBucketPolicyRequest.class))) + .thenReturn(response); + + // Ranger statement (different from IAM) + org.apache.ranger.s3.PolicyStatement rangerStmt = new org.apache.ranger.s3.PolicyStatement(); + rangerStmt.setEffect("Allow"); + rangerStmt.setResource("arn:aws:s3:::ranger-bucket/*"); + rangerStmt.setAction(java.util.Arrays.asList("s3:GetObject")); + List rangerStatements = java.util.Arrays.asList(rangerStmt); + + // Execute + java.util.Set rangerManagedResources = new java.util.HashSet<>(java.util.Arrays.asList("arn:aws:s3:::ranger-bucket/*")); + List result = + serviceDBStore.mergeWithIAMStatements(s3Client, "test-bucket", rangerStatements, rangerManagedResources); + + // Assert: both IAM-only and Ranger statements should be present + Assert.assertEquals(2, result.size()); + // First should be IAM-only statement + Assert.assertEquals("arn:aws:s3:::iam-bucket/*", result.get(0).getResource()); + // Second should be Ranger statement + Assert.assertEquals("arn:aws:s3:::ranger-bucket/*", result.get(1).getResource()); + } + + @Test + public void testMergeWithIAMStatements_OverlappingStatements() throws Exception { + // Setup: mock S3Client with IAM policy that overlaps with Ranger + software.amazon.awssdk.services.s3.S3Client s3Client = Mockito.mock(software.amazon.awssdk.services.s3.S3Client.class); + + // IAM policy with same resource as Ranger + String iamPolicyJson = "{\"Version\":\"2012-10-17\",\"Statement\":[" + + "{\"Effect\":\"Allow\",\"Resource\":\"arn:aws:s3:::shared-bucket/*\",\"Action\":[\"s3:GetObject\"]}" + + "]}"; + + software.amazon.awssdk.services.s3.model.GetBucketPolicyResponse response = + software.amazon.awssdk.services.s3.model.GetBucketPolicyResponse.builder() + .policy(iamPolicyJson) + .build(); + + Mockito.when(s3Client.getBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.GetBucketPolicyRequest.class))) + .thenReturn(response); + + // Ranger statement (same as IAM - should replace) + org.apache.ranger.s3.PolicyStatement rangerStmt = new org.apache.ranger.s3.PolicyStatement(); + rangerStmt.setEffect("Allow"); + rangerStmt.setResource("arn:aws:s3:::shared-bucket/*"); + rangerStmt.setAction(java.util.Arrays.asList("s3:GetObject")); + List rangerStatements = java.util.Arrays.asList(rangerStmt); + + // Execute + java.util.Set rangerManagedResources = new java.util.HashSet<>(java.util.Arrays.asList("arn:aws:s3:::ranger-bucket/*")); + List result = + serviceDBStore.mergeWithIAMStatements(s3Client, "test-bucket", rangerStatements, rangerManagedResources); + + // Assert: only Ranger statement should be present (replaces IAM statement) + Assert.assertEquals(1, result.size()); + Assert.assertEquals("arn:aws:s3:::shared-bucket/*", result.get(0).getResource()); + } + + @Test + public void testProcessPolicies_SingleBucket() throws Exception { + // Setup: Create bucket map with one bucket and one policy + Map>> bucketMap = new HashMap<>(); + Map> policyMap = new HashMap<>(); + + RangerPolicy policy = createS3Policy(1L, "test-policy", "test-bucket/data/*", + java.util.Arrays.asList("user1"), + java.util.Arrays.asList("s3:GetObject")); + + java.util.Set paths = new java.util.HashSet<>(); + paths.add("test-bucket/data/*"); + policyMap.put(policy, paths); + bucketMap.put("test-bucket", policyMap); + + // Mock AWS clients + software.amazon.awssdk.services.s3.S3Client s3Client = Mockito.mock(software.amazon.awssdk.services.s3.S3Client.class); + software.amazon.awssdk.services.iam.IamClient iamClient = Mockito.mock(software.amazon.awssdk.services.iam.IamClient.class); + + // Mock IAM user lookup + software.amazon.awssdk.services.iam.model.GetUserResponse userResponse = + software.amazon.awssdk.services.iam.model.GetUserResponse.builder() + .user(software.amazon.awssdk.services.iam.model.User.builder() + .arn("arn:aws:iam::123456789012:user/user1") + .build()) + .build(); + Mockito.when(iamClient.getUser(Mockito.any(software.amazon.awssdk.services.iam.model.GetUserRequest.class))) + .thenReturn(userResponse); + + // Mock S3 getBucketPolicy to return no existing policy + Mockito.when(s3Client.getBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.GetBucketPolicyRequest.class))) + .thenThrow(software.amazon.awssdk.services.s3.model.NoSuchBucketPolicyException.class); + + // Mock S3 putBucketPolicy + Mockito.when(s3Client.putBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.PutBucketPolicyRequest.class))) + .thenReturn(software.amazon.awssdk.services.s3.model.PutBucketPolicyResponse.builder().build()); + + // Execute: processPolicies should process the bucket map + serviceDBStore.processPolicies(bucketMap, s3Client, iamClient, java.util.Collections.emptySet()); + + // Assert: putBucketPolicy should be called once + Mockito.verify(s3Client, Mockito.times(1)) + .putBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.PutBucketPolicyRequest.class)); + } + + @Test + public void testProcessPolicies_MultipleBuckets() throws Exception { + // Setup: Create bucket map with multiple buckets + Map>> bucketMap = new HashMap<>(); + + // Bucket 1 + Map> policyMap1 = new HashMap<>(); + RangerPolicy policy1 = createS3Policy(1L, "policy-1", "bucket1/data/*", + java.util.Arrays.asList("user1"), + java.util.Arrays.asList("s3:GetObject")); + java.util.Set paths1 = new java.util.HashSet<>(); + paths1.add("bucket1/data/*"); + policyMap1.put(policy1, paths1); + bucketMap.put("bucket1", policyMap1); + + // Bucket 2 + Map> policyMap2 = new HashMap<>(); + RangerPolicy policy2 = createS3Policy(2L, "policy-2", "bucket2/logs/*", + java.util.Arrays.asList("user2"), + java.util.Arrays.asList("s3:PutObject")); + java.util.Set paths2 = new java.util.HashSet<>(); + paths2.add("bucket2/logs/*"); + policyMap2.put(policy2, paths2); + bucketMap.put("bucket2", policyMap2); + + // Mock AWS clients + software.amazon.awssdk.services.s3.S3Client s3Client = Mockito.mock(software.amazon.awssdk.services.s3.S3Client.class); + software.amazon.awssdk.services.iam.IamClient iamClient = Mockito.mock(software.amazon.awssdk.services.iam.IamClient.class); + + // Mock IAM user lookups + software.amazon.awssdk.services.iam.model.GetUserResponse userResponse1 = + software.amazon.awssdk.services.iam.model.GetUserResponse.builder() + .user(software.amazon.awssdk.services.iam.model.User.builder() + .arn("arn:aws:iam::123456789012:user/user1") + .build()) + .build(); + software.amazon.awssdk.services.iam.model.GetUserResponse userResponse2 = + software.amazon.awssdk.services.iam.model.GetUserResponse.builder() + .user(software.amazon.awssdk.services.iam.model.User.builder() + .arn("arn:aws:iam::123456789012:user/user2") + .build()) + .build(); + + Mockito.when(iamClient.getUser(Mockito.any(software.amazon.awssdk.services.iam.model.GetUserRequest.class))) + .thenReturn(userResponse1, userResponse2); + + // Mock S3 getBucketPolicy to return no existing policy + Mockito.when(s3Client.getBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.GetBucketPolicyRequest.class))) + .thenThrow(software.amazon.awssdk.services.s3.model.NoSuchBucketPolicyException.class); + + // Mock S3 putBucketPolicy + Mockito.when(s3Client.putBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.PutBucketPolicyRequest.class))) + .thenReturn(software.amazon.awssdk.services.s3.model.PutBucketPolicyResponse.builder().build()); + + // Execute: processPolicies should process both buckets + serviceDBStore.processPolicies(bucketMap, s3Client, iamClient, java.util.Collections.emptySet()); + + // Assert: putBucketPolicy should be called twice (once for each bucket) + Mockito.verify(s3Client, Mockito.times(2)) + .putBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.PutBucketPolicyRequest.class)); + } + + @Test + public void testProcessPolicies_WithDenyPolicies() throws Exception { + // Setup: Create bucket map with both allow and deny policies + Map>> bucketMap = new HashMap<>(); + Map> policyMap = new HashMap<>(); + + // Create policy with both allow and deny items + RangerPolicy policy = new RangerPolicy(); + policy.setId(1L); + policy.setName("test-policy"); + policy.setService("s3-service"); + policy.setIsEnabled(true); + + // Allow items + List allowItems = new ArrayList<>(); + RangerPolicyItem allowItem = new RangerPolicyItem(); + allowItem.setUsers(java.util.Arrays.asList("user1")); + List allowAccesses = new ArrayList<>(); + RangerPolicyItemAccess allowAccess = new RangerPolicyItemAccess(); + allowAccess.setType("s3:GetObject"); + allowAccess.setIsAllowed(true); + allowAccesses.add(allowAccess); + allowItem.setAccesses(allowAccesses); + allowItems.add(allowItem); + policy.setPolicyItems(allowItems); + + // Deny items + List denyItems = new ArrayList<>(); + RangerPolicyItem denyItem = new RangerPolicyItem(); + denyItem.setUsers(java.util.Arrays.asList("user2")); + List denyAccesses = new ArrayList<>(); + RangerPolicyItemAccess denyAccess = new RangerPolicyItemAccess(); + denyAccess.setType("s3:DeleteObject"); + denyAccess.setIsAllowed(true); + denyAccesses.add(denyAccess); + denyItem.setAccesses(denyAccesses); + denyItems.add(denyItem); + policy.setDenyPolicyItems(denyItems); + + // Resources + Map resources = new HashMap<>(); + RangerPolicyResource pathResource = new RangerPolicyResource(); + pathResource.setValues(java.util.Collections.singletonList("test-bucket/*")); + resources.put("path", pathResource); + policy.setResources(resources); + + java.util.Set paths = new java.util.HashSet<>(); + paths.add("test-bucket/*"); + policyMap.put(policy, paths); + bucketMap.put("test-bucket", policyMap); + + // Mock AWS clients + software.amazon.awssdk.services.s3.S3Client s3Client = Mockito.mock(software.amazon.awssdk.services.s3.S3Client.class); + software.amazon.awssdk.services.iam.IamClient iamClient = Mockito.mock(software.amazon.awssdk.services.iam.IamClient.class); + + // Mock IAM user lookups + software.amazon.awssdk.services.iam.model.GetUserResponse userResponse = + software.amazon.awssdk.services.iam.model.GetUserResponse.builder() + .user(software.amazon.awssdk.services.iam.model.User.builder() + .arn("arn:aws:iam::123456789012:user/user1") + .build()) + .build(); + Mockito.when(iamClient.getUser(Mockito.any(software.amazon.awssdk.services.iam.model.GetUserRequest.class))) + .thenReturn(userResponse); + + // Mock S3 getBucketPolicy to return no existing policy + Mockito.when(s3Client.getBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.GetBucketPolicyRequest.class))) + .thenThrow(software.amazon.awssdk.services.s3.model.NoSuchBucketPolicyException.class); + + // Mock S3 putBucketPolicy + Mockito.when(s3Client.putBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.PutBucketPolicyRequest.class))) + .thenReturn(software.amazon.awssdk.services.s3.model.PutBucketPolicyResponse.builder().build()); + + // Execute: processPolicies should handle both allow and deny items + serviceDBStore.processPolicies(bucketMap, s3Client, iamClient, java.util.Collections.emptySet()); + + // Assert: putBucketPolicy should be called once + Mockito.verify(s3Client, Mockito.times(1)) + .putBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.PutBucketPolicyRequest.class)); + } + + @Test + public void testProcessPolicies_EmptyBucketMap() throws Exception { + // Setup: Empty bucket map + Map>> bucketMap = new HashMap<>(); + + // Mock AWS clients + software.amazon.awssdk.services.s3.S3Client s3Client = Mockito.mock(software.amazon.awssdk.services.s3.S3Client.class); + software.amazon.awssdk.services.iam.IamClient iamClient = Mockito.mock(software.amazon.awssdk.services.iam.IamClient.class); + + // Execute: processPolicies with empty map should not throw + serviceDBStore.processPolicies(bucketMap, s3Client, iamClient, java.util.Collections.emptySet()); + + // Assert: putBucketPolicy should never be called + Mockito.verify(s3Client, Mockito.never()) + .putBucketPolicy(Mockito.any(software.amazon.awssdk.services.s3.model.PutBucketPolicyRequest.class)); + } + @Test public void test54init() throws Exception { serviceDBStore.init();