From a80afd65c4cc6e312de852ed9936cf2593a1f78d Mon Sep 17 00:00:00 2001 From: krishnamuttevi Date: Tue, 8 Sep 2026 18:17:15 +0530 Subject: [PATCH 1/2] RANGER-5696: Added deltaSyncServerType config to optimize delta sync filters for AD/LDAP --- .../process/LdapUserGroupBuilder.java | 21 +++- .../config/UserGroupSyncConfig.java | 13 +++ .../process/TestLdapUserGroupBuilder.java | 106 ++++++++++++++++++ .../src/test/resources/ranger-ugsync-site.xml | 12 ++ 4 files changed, 150 insertions(+), 2 deletions(-) diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java index ae860f2751..dd423197ba 100644 --- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java @@ -122,6 +122,7 @@ public class LdapUserGroupBuilder implements UserGroupSource { private String groupNameAttribute; private String groupCloudIdAttribute; private String currentSyncSource; + private String deltaSyncServerType; private String[] userSearchBase; private String[] groupSearchBase; private Set groupNameSet; @@ -376,6 +377,7 @@ private void setConfig() throws Throwable { currentSyncSource = config.getCurrentSyncSource(); userSearchEnabled = config.isUserSearchEnabled(); groupSearchEnabled = config.isGroupSearchEnabled(); + deltaSyncServerType = config.getDeltaSyncServerType(); ldapUrl = config.getLdapUrl(); ldapBindDn = config.getLdapBindDn(); ldapBindPassword = config.getLdapBindPassword(); @@ -513,11 +515,19 @@ private long getUsers(boolean computeDeletes) throws Throwable { } if (config.isDeltaSyncEnabled()) { - extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(|(uSNChanged>=" + deltaSyncUserTime + ")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z))"; + if ("ad".equalsIgnoreCase(deltaSyncServerType)) { + extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(uSNChanged>=" + deltaSyncUserTime + ")"; + } else if ("ldap".equalsIgnoreCase(deltaSyncServerType)) { + extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z)"; + } else { + extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(|(uSNChanged>=" + deltaSyncUserTime + ")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z))"; + } } else { extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")"; } + LOG.debug("custom userSearchFilter = {}, deltaSyncServerType = {}, extendedUserSearchFilter = {}", userSearchFilter, deltaSyncServerType, extendedUserSearchFilter); + if (userSearchFilter != null && !userSearchFilter.trim().isEmpty()) { String customFilter = userSearchFilter.trim(); @@ -758,11 +768,18 @@ private long getGroups(boolean computeDeletes) throws Throwable { } if (config.isDeltaSyncEnabled()) { - extendedAllGroupsSearchFilter = "(&" + extendedGroupSearchFilter + "(|(uSNChanged>=" + deltaSyncGroupTime + ")(modifyTimestamp>=" + deltaSyncGroupTimeStamp + "Z)))"; + if ("ad".equalsIgnoreCase(deltaSyncServerType)) { + extendedAllGroupsSearchFilter = "(&" + extendedGroupSearchFilter + "(uSNChanged>=" + deltaSyncGroupTime + "))"; + } else if ("ldap".equalsIgnoreCase(deltaSyncServerType)) { + extendedAllGroupsSearchFilter = "(&" + extendedGroupSearchFilter + "(modifyTimestamp>=" + deltaSyncGroupTimeStamp + "Z))"; + } else { + extendedAllGroupsSearchFilter = "(&" + extendedGroupSearchFilter + "(|(uSNChanged>=" + deltaSyncGroupTime + ")(modifyTimestamp>=" + deltaSyncGroupTimeStamp + "Z)))"; + } } else { extendedAllGroupsSearchFilter = "(&" + extendedGroupSearchFilter + ")"; } + LOG.debug("custom groupSearchFilter = {}, deltaSyncServerType = {}, extendedGroupSearchFilter = {}", groupSearchFilter, deltaSyncServerType, extendedGroupSearchFilter); LOG.info("extendedAllGroupsSearchFilter = {}", extendedAllGroupsSearchFilter); for (int ou = 0; ou < groupSearchBase.length; ou++) { diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java index dd62a5b9bc..ec2e1b7cb3 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java @@ -69,6 +69,7 @@ public class UserGroupSyncConfig { public static final String UGSYNC_TEST_RUN_PROP = "ranger.usersync.policymanager.testrun"; /* Other Configs */ public static final String UGSYNC_SERVER_HA_ENABLED_PARAM = "ranger-ugsync.server.ha.enabled"; + public static final String UGSYNC_LDAP_DELTASYNC_SERVER_TYPE = "ranger.usersync.ldap.deltasync.server.type"; public static final String UGSYNC_NAME_VALIDATION_ENABLED = "ranger.usersync.name.validation.enabled"; public static final String UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = "ranger.usersync.syncsource.validation.enabled"; private static final Logger LOG = LoggerFactory.getLogger(UserGroupSyncConfig.class); @@ -805,6 +806,18 @@ public boolean isGroupSearchFirstEnabled() { return groupSearchFirstEnabled; } + public String getDeltaSyncServerType() { + String val = prop.getProperty(UGSYNC_LDAP_DELTASYNC_SERVER_TYPE); + if (val == null || val.trim().isEmpty()) { + return ""; + } else if (val.trim().equalsIgnoreCase("ldap")) { + return "ldap"; + } else if (val.trim().equalsIgnoreCase("ad")) { + return "ad"; + } + return ""; + } + /* Used only for unit testing */ public void setGroupSearchFirstEnabled(boolean groupSearchFirstEnabled) { prop.setProperty(LGSYNC_GROUP_SEARCH_FIRST_ENABLED, String.valueOf(groupSearchFirstEnabled)); diff --git a/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java b/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java index 0d677c3840..f925babb54 100644 --- a/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java +++ b/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java @@ -420,6 +420,112 @@ public void testU_user_config_fields_reflect_properties_and_timestamps_initializ assertEquals(STR_EPOCH, ts.get(b)); } + @Test + void testUG_getUsers_ad_filter_uses_uSNChanged_only() throws Throwable { + resetConfig(); + configureMinimalLdapConfig(); + UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance(); + cfg.setProperty("ranger.usersync.ldap.deltasync", "true"); + cfg.setProperty("ranger.usersync.ldap.deltasync.server.type", "ad"); + + LdapUserGroupBuilder b = new LdapUserGroupBuilder(); + b.init(); + + Field fGroupEnabled = LdapUserGroupBuilder.class.getDeclaredField("groupSearchEnabled"); + fGroupEnabled.setAccessible(true); + fGroupEnabled.setBoolean(b, false); + + Field gutF = LdapUserGroupBuilder.class.getDeclaredField("groupUserTable"); + Field srcUsersF = LdapUserGroupBuilder.class.getDeclaredField("sourceUsers"); + Field srcGroupsF = LdapUserGroupBuilder.class.getDeclaredField("sourceGroups"); + gutF.setAccessible(true); + srcUsersF.setAccessible(true); + srcGroupsF.setAccessible(true); + gutF.set(b, HashBasedTable.create()); + srcUsersF.set(b, new HashMap<>()); + srcGroupsF.set(b, new HashMap<>()); + + final String userDn = "cn=User1000,ou=people,dc=example,dc=com"; + Attributes attrs = new BasicAttributes(true); + attrs.put(new BasicAttribute("cn", "User1000")); + attrs.put(new BasicAttribute("uSNChanged", "2001")); + + class TestSearchResult extends SearchResult { + private final String nameInNs; + + TestSearchResult(String name, Attributes a, String nameInNs) { + super(name, null, a); + this.nameInNs = nameInNs; + } + + @Override + public String getNameInNamespace() { + return nameInNs; + } + } + + final SearchResult sr = new TestSearchResult(userDn, attrs, userDn); + + class SingleEnum implements NamingEnumeration { + private boolean consumed; + + @Override + public SearchResult next() { + consumed = true; + return sr; + } + + @Override + public boolean hasMore() { + return !consumed; + } + + @Override + public void close() {} + + @Override + public boolean hasMoreElements() { + return hasMore(); + } + + @Override + public SearchResult nextElement() { + return next(); + } + } + + try (MockedConstruction mocked = Mockito.mockConstruction(InitialLdapContext.class, (mock, ctx) -> { + Mockito.when(mock.search(Mockito.anyString(), Mockito.anyString(), Mockito.any(SearchControls.class))) + .thenReturn(new SingleEnum()); + Mockito.when(mock.getResponseControls()).thenReturn(null); + })) { + Method getUsers = LdapUserGroupBuilder.class.getDeclaredMethod("getUsers", boolean.class); + getUsers.setAccessible(true); + getUsers.invoke(b, false); + + Field extFilF = LdapUserGroupBuilder.class.getDeclaredField("extendedUserSearchFilter"); + extFilF.setAccessible(true); + String ext = (String) extFilF.get(b); + assertTrue(ext.contains("uSNChanged>="), ext); + assertFalse(ext.contains("modifyTimestamp>="), ext); + } + } + + @Test + void testUF_init_loads_deltaSyncServerType() throws Throwable { + resetConfig(); + configureMinimalLdapConfig(); + UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance(); + cfg.setProperty("ranger.usersync.ldap.deltasync.server.type", "ad"); + + LdapUserGroupBuilder b = new LdapUserGroupBuilder(); + b.init(); + + Field f = LdapUserGroupBuilder.class.getDeclaredField("deltaSyncServerType"); + f.setAccessible(true); + assertEquals("ad", f.get(b)); + } + @Test public void testV_getUsers_processes_user_and_group_memberships() throws Throwable { resetConfig(); diff --git a/ugsync/src/test/resources/ranger-ugsync-site.xml b/ugsync/src/test/resources/ranger-ugsync-site.xml index 5ed504b869..98a80ecf6c 100644 --- a/ugsync/src/test/resources/ranger-ugsync-site.xml +++ b/ugsync/src/test/resources/ranger-ugsync-site.xml @@ -177,4 +177,16 @@ ranger.usersync.group.hierarchylevels 2 + + + ranger.usersync.ldap.deltasync.server.type + + Specifies the LDAP server type for delta sync attribute selection. + Accepted values: + - "AD" : Uses only 'uSNChanged' attribute for delta sync (Active Directory). + - "LDAP" : Uses only 'modifyTimestamp' attribute for delta sync (OpenLDAP). + - "" : Uses both 'uSNChanged' and 'modifyTimestamp' attributes (default behavior). + + + From 0670843dfb95d2d9c8ebd913d1eacaa07bca56d8 Mon Sep 17 00:00:00 2001 From: krishnamuttevi Date: Wed, 9 Sep 2026 13:32:18 +0530 Subject: [PATCH 2/2] RANGER-5696: Added the getDeltaSyncFilter function --- .../process/LdapUserGroupBuilder.java | 39 ++++++++++--------- .../config/UserGroupSyncConfig.java | 13 ------- .../process/TestLdapUserGroupBuilder.java | 15 ------- 3 files changed, 20 insertions(+), 47 deletions(-) diff --git a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java index dd423197ba..e8379b2113 100644 --- a/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java +++ b/ugsync/src/main/java/org/apache/ranger/ldapusersync/process/LdapUserGroupBuilder.java @@ -122,7 +122,6 @@ public class LdapUserGroupBuilder implements UserGroupSource { private String groupNameAttribute; private String groupCloudIdAttribute; private String currentSyncSource; - private String deltaSyncServerType; private String[] userSearchBase; private String[] groupSearchBase; private Set groupNameSet; @@ -377,7 +376,6 @@ private void setConfig() throws Throwable { currentSyncSource = config.getCurrentSyncSource(); userSearchEnabled = config.isUserSearchEnabled(); groupSearchEnabled = config.isGroupSearchEnabled(); - deltaSyncServerType = config.getDeltaSyncServerType(); ldapUrl = config.getLdapUrl(); ldapBindDn = config.getLdapBindDn(); ldapBindPassword = config.getLdapBindPassword(); @@ -515,19 +513,13 @@ private long getUsers(boolean computeDeletes) throws Throwable { } if (config.isDeltaSyncEnabled()) { - if ("ad".equalsIgnoreCase(deltaSyncServerType)) { - extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(uSNChanged>=" + deltaSyncUserTime + ")"; - } else if ("ldap".equalsIgnoreCase(deltaSyncServerType)) { - extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z)"; - } else { - extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")(|(uSNChanged>=" + deltaSyncUserTime + ")(modifyTimestamp>=" + deltaSyncUserTimeStamp + "Z))"; - } + extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")" + getDeltaSyncFilter(deltaSyncUserTime, deltaSyncUserTimeStamp); } else { extendedUserSearchFilter = "(objectclass=" + userObjectClass + ")"; } - LOG.debug("custom userSearchFilter = {}, deltaSyncServerType = {}, extendedUserSearchFilter = {}", userSearchFilter, deltaSyncServerType, extendedUserSearchFilter); - + LOG.debug("custom userSearchFilter = {}, extendedUserSearchFilter = {}", userSearchFilter, extendedUserSearchFilter); + LOG.info("extendedUserSearchFilter = {}", extendedUserSearchFilter); if (userSearchFilter != null && !userSearchFilter.trim().isEmpty()) { String customFilter = userSearchFilter.trim(); @@ -768,18 +760,12 @@ private long getGroups(boolean computeDeletes) throws Throwable { } if (config.isDeltaSyncEnabled()) { - if ("ad".equalsIgnoreCase(deltaSyncServerType)) { - extendedAllGroupsSearchFilter = "(&" + extendedGroupSearchFilter + "(uSNChanged>=" + deltaSyncGroupTime + "))"; - } else if ("ldap".equalsIgnoreCase(deltaSyncServerType)) { - extendedAllGroupsSearchFilter = "(&" + extendedGroupSearchFilter + "(modifyTimestamp>=" + deltaSyncGroupTimeStamp + "Z))"; - } else { - extendedAllGroupsSearchFilter = "(&" + extendedGroupSearchFilter + "(|(uSNChanged>=" + deltaSyncGroupTime + ")(modifyTimestamp>=" + deltaSyncGroupTimeStamp + "Z)))"; - } + extendedAllGroupsSearchFilter = "(&" + extendedGroupSearchFilter + getDeltaSyncFilter(deltaSyncGroupTime, deltaSyncGroupTimeStamp) + ")"; } else { extendedAllGroupsSearchFilter = "(&" + extendedGroupSearchFilter + ")"; } - LOG.debug("custom groupSearchFilter = {}, deltaSyncServerType = {}, extendedGroupSearchFilter = {}", groupSearchFilter, deltaSyncServerType, extendedGroupSearchFilter); + LOG.debug("custom groupSearchFilter = {}, extendedGroupSearchFilter = {}", groupSearchFilter, extendedGroupSearchFilter); LOG.info("extendedAllGroupsSearchFilter = {}", extendedAllGroupsSearchFilter); for (int ou = 0; ou < groupSearchBase.length; ou++) { @@ -926,6 +912,21 @@ private long getGroups(boolean computeDeletes) throws Throwable { return highestdeltaSyncGroupTime; } + private String getDeltaSyncFilter(long usnChangedTime, String modifyTimestamp) { + String result = ""; + String serverType = config.getProperty("ranger.usersync.ldap.deltasync.server.type", "").trim(); + + if (serverType.equalsIgnoreCase("ad")) { + result = "(uSNChanged>=" + usnChangedTime + ")"; + } else if (serverType.equalsIgnoreCase("ldap")) { + result = "(modifyTimestamp>=" + modifyTimestamp + "Z)"; + } else { + result = "(|(uSNChanged>=" + usnChangedTime + ")(modifyTimestamp>=" + modifyTimestamp + "Z))"; + } + + return result; + } + private void goUpGroupHierarchy(Set groups, int groupHierarchyLevels, String groupSName) { if (groupHierarchyLevels <= 0 || groups.isEmpty()) { return; diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java index ec2e1b7cb3..dd62a5b9bc 100644 --- a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java +++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java @@ -69,7 +69,6 @@ public class UserGroupSyncConfig { public static final String UGSYNC_TEST_RUN_PROP = "ranger.usersync.policymanager.testrun"; /* Other Configs */ public static final String UGSYNC_SERVER_HA_ENABLED_PARAM = "ranger-ugsync.server.ha.enabled"; - public static final String UGSYNC_LDAP_DELTASYNC_SERVER_TYPE = "ranger.usersync.ldap.deltasync.server.type"; public static final String UGSYNC_NAME_VALIDATION_ENABLED = "ranger.usersync.name.validation.enabled"; public static final String UGSYNC_SYNC_SOURCE_VALIDATION_ENABLED = "ranger.usersync.syncsource.validation.enabled"; private static final Logger LOG = LoggerFactory.getLogger(UserGroupSyncConfig.class); @@ -806,18 +805,6 @@ public boolean isGroupSearchFirstEnabled() { return groupSearchFirstEnabled; } - public String getDeltaSyncServerType() { - String val = prop.getProperty(UGSYNC_LDAP_DELTASYNC_SERVER_TYPE); - if (val == null || val.trim().isEmpty()) { - return ""; - } else if (val.trim().equalsIgnoreCase("ldap")) { - return "ldap"; - } else if (val.trim().equalsIgnoreCase("ad")) { - return "ad"; - } - return ""; - } - /* Used only for unit testing */ public void setGroupSearchFirstEnabled(boolean groupSearchFirstEnabled) { prop.setProperty(LGSYNC_GROUP_SEARCH_FIRST_ENABLED, String.valueOf(groupSearchFirstEnabled)); diff --git a/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java b/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java index f925babb54..0fae39013c 100644 --- a/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java +++ b/ugsync/src/test/java/org/apache/ranger/ldapusersync/process/TestLdapUserGroupBuilder.java @@ -511,21 +511,6 @@ public SearchResult nextElement() { } } - @Test - void testUF_init_loads_deltaSyncServerType() throws Throwable { - resetConfig(); - configureMinimalLdapConfig(); - UserGroupSyncConfig cfg = UserGroupSyncConfig.getInstance(); - cfg.setProperty("ranger.usersync.ldap.deltasync.server.type", "ad"); - - LdapUserGroupBuilder b = new LdapUserGroupBuilder(); - b.init(); - - Field f = LdapUserGroupBuilder.class.getDeclaredField("deltaSyncServerType"); - f.setAccessible(true); - assertEquals("ad", f.get(b)); - } - @Test public void testV_getUsers_processes_user_and_group_memberships() throws Throwable { resetConfig();