Browse Source

Ticket 50544 - OpenLDAP syncrepl compatability

Bug Description: Some customers have asked for the ability to sync
openldap fro 389-ds in a read only mode. OpenLDAP's syncrepl
functionality is slightly different to what our module expected,
requiring changes to be made.

Fix Description: This fixes a number of syncrepl issues within
our plugin, works around a number of deviations from OpenLDAP's
syncrepl client, adds tests, and the needed schema to allow
OpenLDAP to sync from 389-ds.

Outstanding issue is that when the EntryUUID plugin is enabled, it
can confuse OpenLDAP, so a subsequent PR will address that issue.

Note, the provided tests require a fix to python-ldap, so you may
not be able to run these tests yet. See:
https://github.com/python-ldap/python-ldap/pull/351

https://pagure.io/389-ds-base/issue/50544

Author: William Brown <william@blackhats.net.au>

Review by: mreynolds (Thanks!)
tags/upstream/1.4.4.4
William Brown 1 year ago
parent
commit
8f3887e08c
12 changed files with 868 additions and 71 deletions
  1. +47
    -0
      LICENSE.openldap
  2. +1
    -0
      Makefile.am
  3. +163
    -0
      dirsrvtests/tests/suites/syncrepl_plugin/__init__.py
  4. +60
    -0
      dirsrvtests/tests/suites/syncrepl_plugin/basic_test.py
  5. +64
    -0
      dirsrvtests/tests/suites/syncrepl_plugin/openldap_test.py
  6. +208
    -0
      ldap/schema/dsee.schema
  7. +4
    -2
      ldap/servers/plugins/sync/sync.h
  8. +85
    -28
      ldap/servers/plugins/sync/sync_refresh.c
  9. +206
    -41
      ldap/servers/plugins/sync/sync_util.c
  10. +2
    -0
      src/lib389/lib389/cli_conf/plugin.py
  11. +15
    -0
      src/lib389/lib389/cli_conf/plugins/contentsync.py
  12. +13
    -0
      src/lib389/lib389/plugins.py

+ 47
- 0
LICENSE.openldap View File

@@ -0,0 +1,47 @@
The OpenLDAP Public License
Version 2.8, 17 August 2003

Redistribution and use of this software and associated documentation
("Software"), with or without modification, are permitted provided
that the following conditions are met:

1. Redistributions in source form must retain copyright statements
and notices,

2. Redistributions in binary form must reproduce applicable copyright
statements and notices, this list of conditions, and the following
disclaimer in the documentation and/or other materials provided
with the distribution, and

3. Redistributions must contain a verbatim copy of this document.

The OpenLDAP Foundation may revise this license from time to time.
Each revision is distinguished by a version number. You may use
this Software under terms of this license revision or under the
terms of any subsequent revision of the license.

THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS
CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S)
OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

The names of the authors and copyright holders must not be used in
advertising or otherwise to promote the sale, use or other dealing
in this Software without specific, written prior permission. Title
to copyright in this Software shall at all times remain with copyright
holders.

OpenLDAP is a registered trademark of the OpenLDAP Foundation.

Copyright 1999-2003 The OpenLDAP Foundation, Redwood City,
California, USA. All Rights Reserved. Permission to copy and
distribute verbatim copies of this document is granted.

+ 1
- 0
Makefile.am View File

@@ -725,6 +725,7 @@ sampledata_DATA = ldap/admin/src/scripts/DSSharedLib \
$(srcdir)/ldap/schema/60rfc4876.ldif \
$(srcdir)/ldap/schema/60samba.ldif \
$(srcdir)/ldap/schema/60sendmail.ldif \
$(srcdir)/ldap/schema/dsee.schema \
$(LIBPRESENCE_SCHEMA)

systemschema_DATA = $(srcdir)/ldap/schema/00core.ldif \


+ 163
- 0
dirsrvtests/tests/suites/syncrepl_plugin/__init__.py View File

@@ -0,0 +1,163 @@
# --- BEGIN COPYRIGHT BLOCK ---
# Copyright (C) 2020 William Brown <william@blackhats.net.au>
# All rights reserved.
#
# License: GPL (version 3 or any later version).
# See LICENSE for details.
# --- END COPYRIGHT BLOCK ---

import logging
import ldap
import time
from ldap.syncrepl import SyncreplConsumer
import pytest
from lib389 import DirSrv
from lib389.idm.user import nsUserAccounts, UserAccounts
from lib389.topologies import topology_st as topology
from lib389.paths import Paths
from lib389.utils import ds_is_older
from lib389.plugins import RetroChangelogPlugin, ContentSyncPlugin
from lib389._constants import *

log = logging.getLogger(__name__)

class ISyncRepl(DirSrv, SyncreplConsumer):
"""
This implements a test harness for checking syncrepl, and allowing us to check various actions or
behaviours. During a "run" it stores the results in it's instance, so that they can be inspected
later to ensure that syncrepl worked as expected.
"""
def __init__(self, inst, openldap=False):
self.inst = inst
self.msgid = None

self.last_cookie = None
self.next_cookie = None
self.cookie = None
self.openldap = openldap
if self.openldap:
# In openldap mode, our initial cookie needs to be a rid.
self.cookie = "rid=123"
self.delete = []
self.present = []
self.entries = {}

super().__init__()

def result4(self, *args, **kwargs):
return self.inst.result4(*args, **kwargs, escapehatch='i am sure')

def search_ext(self, *args, **kwargs):
return self.inst.search_ext(*args, **kwargs, escapehatch='i am sure')

def syncrepl_search(self, base=DEFAULT_SUFFIX, scope=ldap.SCOPE_SUBTREE, mode='refreshOnly', cookie=None, **search_args):
# Wipe the last result set.
self.delete = []
self.present = []
self.entries = {}
self.next_cookie = None
# Start the sync
# If cookie is none, will call "get_cookie" we have.
self.msgid = super().syncrepl_search(base, scope, mode, cookie, **search_args)
log.debug(f'syncrepl_search -> {self.msgid}')
assert self.msgid is not None

def syncrepl_complete(self):
log.debug(f'syncrepl_complete -> {self.msgid}')
assert self.msgid is not None
# Loop until the operation is complete.
while super().syncrepl_poll(msgid=self.msgid) is True:
pass
assert self.next_cookie is not None
self.last_cookie = self.cookie
self.cookie = self.next_cookie

def check_cookie(self):
assert self.last_cookie != self.cookie

def syncrepl_set_cookie(self, cookie):
log.debug(f'set_cookie -> {cookie}')
if self.openldap:
assert self.cookie.startswith("rid=123")
self.next_cookie = cookie

def syncrepl_get_cookie(self):
log.debug('get_cookie -> %s' % self.cookie)
if self.openldap:
assert self.cookie.startswith("rid=123")
return self.cookie

def syncrepl_present(self, uuids, refreshDeletes=False):
log.debug(f'=====> refdel -> {refreshDeletes} uuids -> {uuids}')
if uuids is not None:
self.present = self.present + uuids

def syncrepl_delete(self, uuids):
log.debug(f'delete -> {uuids}')
self.delete = uuids

def syncrepl_entry(self, dn, attrs, uuid):
log.debug(f'entry -> {dn}')
self.entries[dn] = (uuid, attrs)

def syncrepl_refreshdone(self):
log.debug('refreshdone')

def syncstate_assert(st, sync):
# How many entries do we have?
r = st.search_ext_s(
base=DEFAULT_SUFFIX,
scope=ldap.SCOPE_SUBTREE,
filterstr='(objectClass=*)',
attrsonly=1,
escapehatch='i am sure'
)

# Initial sync
log.debug("*test* initial")
sync.syncrepl_search()
sync.syncrepl_complete()
# check we caught them all
assert len(r) == len(sync.entries.keys())
assert len(r) == len(sync.present)
assert 0 == len(sync.delete)

# Add a new entry

account = nsUserAccounts(st, DEFAULT_SUFFIX).create_test_user()
# Check
log.debug("*test* add")
sync.syncrepl_search()
sync.syncrepl_complete()
sync.check_cookie()
assert 1 == len(sync.entries.keys())
assert 1 == len(sync.present)
assert 0 == len(sync.delete)

# Mod
account.replace('description', 'change')
# Check
log.debug("*test* mod")
sync.syncrepl_search()
sync.syncrepl_complete()
sync.check_cookie()
assert 1 == len(sync.entries.keys())
assert 1 == len(sync.present)
assert 0 == len(sync.delete)

## Delete
account.delete()

# Check
log.debug("*test* del")
sync.syncrepl_search()
sync.syncrepl_complete()
# In a delete, the cookie isn't updated (?)
sync.check_cookie()
log.debug(f'{sync.entries.keys()}')
log.debug(f'{sync.present}')
log.debug(f'{sync.delete}')
assert 0 == len(sync.entries.keys())
assert 0 == len(sync.present)
assert 1 == len(sync.delete)


+ 60
- 0
dirsrvtests/tests/suites/syncrepl_plugin/basic_test.py View File

@@ -0,0 +1,60 @@
# --- BEGIN COPYRIGHT BLOCK ---
# Copyright (C) 2020 William Brown <william@blackhats.net.au>
# All rights reserved.
#
# License: GPL (version 3 or any later version).
# See LICENSE for details.
# --- END COPYRIGHT BLOCK ---

import logging
import ldap
import time
from ldap.syncrepl import SyncreplConsumer
import pytest
from lib389 import DirSrv
from lib389.idm.user import nsUserAccounts, UserAccounts
from lib389.topologies import topology_st as topology
from lib389.paths import Paths
from lib389.utils import ds_is_older
from lib389.plugins import RetroChangelogPlugin, ContentSyncPlugin
from lib389._constants import *

from . import ISyncRepl, syncstate_assert

default_paths = Paths()
pytestmark = pytest.mark.tier1

log = logging.getLogger(__name__)

def test_syncrepl_basic(topology):
""" Test basic functionality of the SyncRepl interface

:id: f9fea826-8ae2-412a-8e88-b8e0ba939b06

:setup: Standalone instance

:steps:
1. Enable Retro Changelog
2. Enable Syncrepl
3. Run the syncstate test to check refresh, add, delete, mod.

:expectedresults:
1. Success
1. Success
1. Success
"""
st = topology.standalone
# Enable RetroChangelog.
rcl = RetroChangelogPlugin(st)
rcl.enable()
# Set the default targetid
rcl.replace('nsslapd-attribute', 'nsuniqueid:targetUniqueId')
# Enable sync repl
csp = ContentSyncPlugin(st)
csp.enable()
# Restart DS
st.restart()
# Setup the syncer
sync = ISyncRepl(st)
# Run the checks
syncstate_assert(st, sync)

+ 64
- 0
dirsrvtests/tests/suites/syncrepl_plugin/openldap_test.py View File

@@ -0,0 +1,64 @@
# Copyright (C) 2020 William Brown <william@blackhats.net.au>
# All rights reserved.
#
# License: GPL (version 3 or any later version).
# See LICENSE for details.
# --- END COPYRIGHT BLOCK ---

import logging
import ldap
import time
from ldap.syncrepl import SyncreplConsumer
import pytest
from lib389 import DirSrv
from lib389.idm.user import nsUserAccounts, UserAccounts
from lib389.topologies import topology_st as topology
from lib389.paths import Paths
from lib389.utils import ds_is_older
from lib389.plugins import RetroChangelogPlugin, ContentSyncPlugin
from lib389._constants import *

from . import ISyncRepl, syncstate_assert

default_paths = Paths()
pytestmark = pytest.mark.tier1

log = logging.getLogger(__name__)

@pytest.mark.skipif(ds_is_older('1.4.4.0'), reason="Sync repl does not support openldap compat in older versions")
def test_syncrepl_openldap(topology):
""" Test basic functionality of the openldap syncrepl
compatability handler.

:id: 03039178-2cc6-40bd-b32c-7d6de108828b

:setup: Standalone instance

:steps:
1. Enable Retro Changelog
2. Enable Syncrepl
3. Run the syncstate test to check refresh, add, delete, mod.

:expectedresults:
1. Success
1. Success
1. Success
"""
st = topology.standalone
# Enable RetroChangelog.
rcl = RetroChangelogPlugin(st)
rcl.enable()
# Set the default targetid
rcl.replace('nsslapd-attribute', 'nsuniqueid:targetUniqueId')
# Enable sync repl
csp = ContentSyncPlugin(st)
csp.enable()
# Restart DS
st.restart()
# log.error("+++++++++++")
# time.sleep(60)
# Setup the syncer
sync = ISyncRepl(st, openldap=True)
# Run the checks
syncstate_assert(st, sync)


+ 208
- 0
ldap/schema/dsee.schema View File

@@ -0,0 +1,208 @@
# $OpenLDAP$
## This work is part of OpenLDAP Software <http://www.openldap.org/>.
##
## Copyright 2019-2020 The OpenLDAP Foundation.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## <http://www.OpenLDAP.org/license.html>.

# This file is provided for informational purposes only.

# These definitions are from Sun DSEE 7's cn=schema subentry.
# None of the attributes had matching rules defined; we've
# inserted usable ones as needed.

# Some of these attributes are defined with NO-USER-MODIFICATION,
# but slapd won't load such definitions from user-modifiable schema
# files. So that designation has been removed, and commented accordingly.

objectidentifier NetscapeRoot 2.16.840.1.113730
objectidentifier NetscapeDS NetscapeRoot:3
objectidentifier NSDSat NetscapeDS:1
objectidentifier NSDSoc NetscapeDS:2
objectidentifier SunRoot 1.3.6.1.4.1.42
objectidentifier SunDS SunRoot:2.27

attributetype ( 1.2.840.113556.1.2.102
NAME 'memberOf'
DESC 'Group that the entry belongs to'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.12
X-ORIGIN 'Netscape Delegated Administrator' )

attributetype ( NSDSat:9999
NAME 'entryId'
DESC 'Supplier Internal Id'
EQUALITY integerMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.27
X-ORIGIN 'Changelog Internet Draft' )

attributetype ( 1.3.1.1.4.1.453.16.2.103
NAME 'numSubordinates'
DESC 'Number of Subordinate Entries from Supplier'
EQUALITY integerMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.27
X-ORIGIN 'Changelog Internet Draft' )

attributetype ( NSDSat:55
NAME 'aci'
DESC 'NSDS ACI'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
X-ORIGIN 'Changelog Internet Draft' )

attributetype ( NSDSat:9998
NAME 'parentId'
DESC 'Supplier Internal Id'
EQUALITY integerMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.27
X-ORIGIN 'Changelog Internet Draft' )

attributetype ( NSDSat:610
NAME 'nsAccountLock'
DESC 'Operational attribute for Account Inactivation'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
SINGLE-VALUE
X-ORIGIN '389 Directory Server Project' )

attributetype ( NSDSat:2343
name 'legalName'
DESC 'An individuals legalName'
EQUALITY caseIgnoreMatch
SUBSTR caseIgnoreSubstringsMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
SINGLE-VALUE
X-ORIGIN '389 Directory Server Project' )

attributetype ( NSDSat:2337
NAME 'nsCertSubjectDN'
DESC 'An x509 DN from a certificate used to map during a TLS bind process'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.12
X-ORIGIN '389 Directory Server Project' )

attributetype ( NSDSat:2111
NAME 'tombstoneNumSubordinates'
DESC 'count of immediate subordinates for tombstone entries'
EQUALITY integerMatch
ORDERING integerOrderingMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.27
SINGLE-VALUE
X-ORIGIN '389 directory server' )

attributetype ( NSDSat:2342
NAME 'nsSshPublicKey'
DESC 'An nsSshPublicKey record'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
X-ORIGIN '389 Directory Server Project' )

attributetype ( NSDSat:5
NAME 'changeNumber'
DESC 'Changelog attribute type'
EQUALITY integerMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.27
X-ORIGIN 'Changelog Internet Draft' )

attributetype ( NSDSat:6
NAME 'targetDn'
DESC 'Changelog attribute type'
EQUALITY distinguishedNameMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.12
X-ORIGIN 'Changelog Internet Draft' )

attributetype ( NSDSat:7
NAME 'changeType'
DESC 'Changelog attribute type'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
X-ORIGIN 'Changelog Internet Draft' )

# They claim Binary syntax but it's really octetString
attributetype ( NSDSat:8
NAME 'changes'
DESC 'Changelog attribute type'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.5
X-ORIGIN 'Changelog Internet Draft' )

attributetype ( NSDSat:9
NAME 'newRdn'
DESC 'Changelog attribute type'
EQUALITY distinguishedNameMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.12
X-ORIGIN 'Changelog Internet Draft' )

attributetype ( NSDSat:10
NAME 'deleteOldRdn'
DESC 'Changelog attribute type'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.7
X-ORIGIN 'Changelog Internet Draft' )

attributetype ( NSDSat:11
NAME 'newSuperior'
DESC 'Changelog attribute type'
EQUALITY distinguishedNameMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.12
X-ORIGIN 'Changelog Internet Draft' )

# should be generalizedTime, but they used directoryString instead...
attributeType ( NSDSat:77
NAME 'changeTime'
DESC 'Sun ONE defined attribute type'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
X-ORIGIN 'Sun ONE Directory Server' )

# These are UUIDs, but (of course) hyphenated differently than ours.
# NO-USER-MODIFICATION
attributetype ( NSDSat:542
NAME 'nsUniqueId'
DESC 'Sun ONE defined attribute type'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
SINGLE-VALUE
X-ORIGIN 'Sun ONE Directory Server' )

# NO-USER-MODIFICATION
attributeype ( SunDS:9.1.596
NAME 'targetUniqueId'
DESC 'RetroChangelog attribute type'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
SINGLE-VALUE
X-ORIGIN 'Sun Directory Server' )

objectclass ( NSDSoc:1
NAME 'changeLogEntry'
DESC 'LDAP changelog objectclass'
SUP top STRUCTURAL
MUST ( targetDn $ changeTime $ changeNumber $ changeType )
MAY ( changes $ newRdn $ deleteOldRdn $ newSuperior )
X-ORIGIN 'Changelog Internet Draft' )

objectclass ( NSDSoc:333
NAME 'nsPerson'
DESC 'A representation of a person in a directory server'
SUP top STRUCTURAL
MUST ( displayName $ cn )
MAY ( userPassword $ seeAlso $ description $ legalName $ mail $ preferredLanguage )
X-ORIGIN '389 Directory Server Project' )

objectclass ( NSDSoc:331
NAME 'nsAccount'
DESC 'A representation of a binding user in a directory server'
SUP top AUXILIARY
MAY ( userCertificate $ nsCertSubjectDN $ nsSshPublicKey $ userPassword $ nsAccountLock )
X-ORIGIN '389 Directory Server Project' )

objectclass ( NSDSoc:334
NAME 'nsOrgPerson'
DESC 'A representation of an org person in directory server. See also inetOrgPerson.'
SUP top AUXILIARY
MAY ( businessCategory $ carLicense $ departmentNumber $ employeeNumber $ employeeType $ homePhone $ homePostalAddress $ initials $ jpegPhoto $ labeledURI $ manager $ mobile $ o $ pager $ photo $ roomNumber $ uid $ userCertificate $ telephoneNumber $ x500uniqueIdentifier $ userSMIMECertificate $ userPKCS12 )
X-ORIGIN '389 Directory Server Project' )

objectclass ( NSDSoc:329
NAME 'nsMemberOf'
DESC 'Allow memberOf assignment on groups for nesting and users'
SUP top AUXILIARY
MAY ( memberOf )
X-ORIGIN '389 Directory Server Project' )

+ 4
- 2
ldap/servers/plugins/sync/sync.h View File

@@ -16,6 +16,7 @@

#include <stdio.h>
#include <string.h>
#include <stdbool.h>
#include "slapi-plugin.h"
#include "slapi-private.h"

@@ -44,6 +45,7 @@ typedef struct sync_cookie
char *cookie_client_signature;
char *cookie_server_signature;
unsigned long cookie_change_info;
bool openldap_compat;
} Sync_Cookie;

typedef struct sync_update
@@ -83,9 +85,9 @@ int sync_intermediate_msg(Slapi_PBlock *pb, int tag, Sync_Cookie *cookie, char *
int sync_result_msg(Slapi_PBlock *pb, Sync_Cookie *cookie);
int sync_result_err(Slapi_PBlock *pb, int rc, char *msg);

Sync_Cookie *sync_cookie_create(Slapi_PBlock *pb);
Sync_Cookie *sync_cookie_create(Slapi_PBlock *pb, Sync_Cookie *client_cookie);
void sync_cookie_update(Sync_Cookie *cookie, Slapi_Entry *ec);
Sync_Cookie *sync_cookie_parse(char *cookie);
Sync_Cookie *sync_cookie_parse(char *cookie, bool *cookie_refresh);
int sync_cookie_isvalid(Sync_Cookie *testcookie, Sync_Cookie *refcookie);
void sync_cookie_free(Sync_Cookie **freecookie);
char *sync_cookie2str(Sync_Cookie *cookie);


+ 85
- 28
ldap/servers/plugins/sync/sync_refresh.c View File

@@ -66,8 +66,9 @@ sync_srch_refresh_pre_search(Slapi_PBlock *pb)
slapi_pblock_get(pb, SLAPI_REQCONTROLS, &requestcontrols);
if (slapi_control_present(requestcontrols, LDAP_CONTROL_SYNC, &psbvp, NULL)) {
char *cookie = NULL;
int mode = 1;
int refresh = 0;
int32_t mode = 1;
int32_t refresh = 0;
bool cookie_refresh = 0;

if (sync_parse_control_value(psbvp, &mode,
&refresh, &cookie) != LDAP_SUCCESS) {
@@ -83,12 +84,22 @@ sync_srch_refresh_pre_search(Slapi_PBlock *pb)
}

if (mode == 1 || mode == 3) {

/* we need to return a cookie in the result message
/*
* OpenLDAP violates rfc4533 by sending a "rid=" in it's initial cookie sync, even
* when using their changelog mode. As a result, we parse the cookie to handle this
* shenangians to determine if this is valid.
*/
client_cookie = sync_cookie_parse(cookie, &cookie_refresh);
/*
* we need to return a cookie in the result message
* indicating a state to be used in future sessions
* as starting point - create it now
* as starting point - create it now. We need to provide
* the client_cookie so we understand if we are in
* openldap mode or not, and to get the 'rid' of the
* consumer.
*/
session_cookie = sync_cookie_create(pb);
session_cookie = sync_cookie_create(pb, client_cookie);
PR_ASSERT(session_cookie);
/*
* if mode is persist we need to setup the persit handler
* to catch the mods while the refresh is done
@@ -104,7 +115,7 @@ sync_srch_refresh_pre_search(Slapi_PBlock *pb)
}
}
/*
* now handl the refresh request
* now handle the refresh request
* there are two scenarios
* 1. no cookie is provided this means send all entries matching the search request
* 2. a cookie is provided: send all entries changed since the cookie was issued
@@ -112,31 +123,34 @@ sync_srch_refresh_pre_search(Slapi_PBlock *pb)
* -- return e-syncRefreshRequired if the data referenced in the cookie are no
* longer in the history
*/
if (cookie) {
if ((client_cookie = sync_cookie_parse(cookie)) &&
sync_cookie_isvalid(client_cookie, session_cookie)) {
if (!cookie_refresh) {
if (sync_cookie_isvalid(client_cookie, session_cookie)) {
rc = sync_refresh_update_content(pb, client_cookie, session_cookie);
if (rc == 0)
if (rc == 0) {
entries_sent = 1;
if (sync_persist)
}
if (sync_persist) {
rc = sync_intermediate_msg(pb, LDAP_TAG_SYNC_REFRESH_DELETE, session_cookie, NULL);
else
} else {
rc = sync_result_msg(pb, session_cookie);
}
} else {
rc = E_SYNC_REFRESH_REQUIRED;
sync_result_err(pb, rc, "Invalid session cookie");
}
} else {
rc = sync_refresh_initial_content(pb, sync_persist, tid, session_cookie);
if (rc == 0 && !sync_persist)
if (rc == 0 && !sync_persist) {
/* maintained in postop code */
session_cookie = NULL;
}
/* if persis it will be handed over to persist code */
}

if (rc) {
if (sync_persist)
if (sync_persist) {
sync_persist_terminate(tid);
}
goto error_return;
} else if (sync_persist) {
Slapi_Operation *operation;
@@ -194,7 +208,38 @@ sync_srch_refresh_post_search(Slapi_PBlock *pb)
if (info->send_flag & SYNC_FLAG_ADD_DONE_CTRL) {
LDAPControl **ctrl = (LDAPControl **)slapi_ch_calloc(2, sizeof(LDAPControl *));
char *cookiestr = sync_cookie2str(info->cookie);
sync_create_sync_done_control(&ctrl[0], 0, cookiestr);
/*
* RFC4533
* If refreshDeletes of syncDoneValue is FALSE, the new copy includes
* all changed entries returned by the reissued Sync Operation, as well
* as all unchanged entries identified as being present by the reissued
* Sync Operation, but whose content is provided by the previous Sync
* Operation. The unchanged entries not identified as being present are
* deleted from the client content. They had been either deleted,
* moved, or otherwise scoped-out from the content.
*
* If refreshDeletes of syncDoneValue is TRUE, the new copy includes all
* changed entries returned by the reissued Sync Operation, as well as
* all other entries of the previous copy except for those that are
* identified as having been deleted from the content.
*
* Confused yet? Don't worry so am I. I have no idea what this means or
* what it will do. The best I can see from wireshark is that if refDel is
* false, then anything *not* present will be purged from the change that
* was supplied. Which probably says a lot about how confusing syncrepl is
* that we've hardcoded this to false for literally years and no one has
* complained, probably because every client is broken in their own ways
* as no one can actually interpret that dense statement above.
*
* Point is, if we set refresh to true for openldap mode, it works, and if
* it's false, the moment we send a single intermediate delete message, we
* delete literally everything 🔥.
*/
if (info->cookie->openldap_compat) {
sync_create_sync_done_control(&ctrl[0], 1, cookiestr);
} else {
sync_create_sync_done_control(&ctrl[0], 0, cookiestr);
}
slapi_pblock_set(pb, SLAPI_RESCONTROLS, ctrl);
slapi_ch_free((void **)&cookiestr);
}
@@ -254,9 +299,21 @@ sync_refresh_update_content(Slapi_PBlock *pb, Sync_Cookie *client_cookie, Sync_C
Slapi_PBlock *seq_pb;
char *filter;
Sync_CallBackData cb_data;
int rc;
int chg_count = server_cookie->cookie_change_info -
client_cookie->cookie_change_info + 1;
int rc = LDAP_SUCCESS;
PR_ASSERT(client_cookie);

/*
* We have nothing to send, move along.
* Should be caught by cookie is valid though if the server < client, but if
* they are equal, we return.
*/
PR_ASSERT(server_cookie->cookie_change_info >= client_cookie->cookie_change_info);
if (server_cookie->cookie_change_info == client_cookie->cookie_change_info) {
return rc;
}

int chg_count = (server_cookie->cookie_change_info - client_cookie->cookie_change_info) + 1;
PR_ASSERT(chg_count > 0);

cb_data.cb_updates = (Sync_UpdateNode *)slapi_ch_calloc(chg_count, sizeof(Sync_UpdateNode));

@@ -581,21 +638,21 @@ sync_read_entry_from_changelog(Slapi_Entry *cl_entry, void *cb_data)
void
sync_send_deleted_entries(Slapi_PBlock *pb, Sync_UpdateNode *upd, int chg_count, Sync_Cookie *cookie)
{
char *syncUUIDs[SYNC_MAX_DELETED_UUID_BATCH + 1];
int uuid_index = 0;
int index, i;
char *syncUUIDs[SYNC_MAX_DELETED_UUID_BATCH + 1] = {0};
size_t uuid_index = 0;

syncUUIDs[0] = NULL;
for (index = 0; index < chg_count; index++) {
for (size_t index = 0; index < chg_count; index++) {
if (upd[index].upd_chgtype == LDAP_REQ_DELETE &&
upd[index].upd_uuid) {
if (uuid_index < SYNC_MAX_DELETED_UUID_BATCH) {
syncUUIDs[uuid_index++] = sync_nsuniqueid2uuid(upd[index].upd_uuid);
syncUUIDs[uuid_index] = sync_nsuniqueid2uuid(upd[index].upd_uuid);
uuid_index++;
} else {
/* max number of uuids to be sent in one sync info message */
syncUUIDs[uuid_index] = NULL;
sync_intermediate_msg(pb, LDAP_TAG_SYNC_ID_SET, cookie, &syncUUIDs[0]);
for (i = 0; i < uuid_index; i++) {
sync_intermediate_msg(pb, LDAP_TAG_SYNC_ID_SET, cookie, (char **)&syncUUIDs);
for (size_t i = 0; i < uuid_index; i++) {
slapi_ch_free((void **)&syncUUIDs[i]);
syncUUIDs[i] = NULL;
}
@@ -607,8 +664,8 @@ sync_send_deleted_entries(Slapi_PBlock *pb, Sync_UpdateNode *upd, int chg_count,
if (uuid_index > 0 && syncUUIDs[uuid_index - 1]) {
/* more entries to send */
syncUUIDs[uuid_index] = NULL;
sync_intermediate_msg(pb, LDAP_TAG_SYNC_ID_SET, cookie, &syncUUIDs[0]);
for (i = 0; i < uuid_index; i++) {
sync_intermediate_msg(pb, LDAP_TAG_SYNC_ID_SET, cookie, (char **)&syncUUIDs);
for (size_t i = 0; i < uuid_index; i++) {
slapi_ch_free((void **)&syncUUIDs[i]);
syncUUIDs[i] = NULL;
}


+ 206
- 41
ldap/servers/plugins/sync/sync_util.c View File

@@ -12,6 +12,11 @@ static struct berval *create_syncinfo_value(int type, const char *cookie, const
static char *sync_cookie_get_server_info(Slapi_PBlock *pb);
static char *sync_cookie_get_client_info(Slapi_PBlock *pb);

static void sync_ulong2olcsn(unsigned long chgnr, char *buf);
static unsigned long sync_olcsn2ulong(char *csn);

#define CSN_OFFSET 4102448461

/*
* Parse the value from an LDAPv3 sync request control. They look
* like this:
@@ -191,14 +196,14 @@ sync_create_sync_done_control(LDAPControl **ctrlp, int refresh, char *cookie)
if (cookie) {
if ((rc = ber_printf(ber, "{s", cookie)) != -1) {
if (refresh) {
rc = ber_printf(ber, "e}", refresh);
rc = ber_printf(ber, "b}", refresh);
} else {
rc = ber_printf(ber, "}");
}
}
} else {
if (refresh) {
rc = ber_printf(ber, "{e}", refresh);
rc = ber_printf(ber, "{b}", refresh);
} else {
rc = ber_printf(ber, "{}");
}
@@ -229,10 +234,18 @@ sync_cookie2str(Sync_Cookie *cookie)
char *cookiestr = NULL;

if (cookie) {
cookiestr = slapi_ch_smprintf("%s#%s#%lu",
cookie->cookie_server_signature,
cookie->cookie_client_signature,
cookie->cookie_change_info);
if (cookie->openldap_compat) {
char buf[16] = {0};
sync_ulong2olcsn(cookie->cookie_change_info, buf);
cookiestr = slapi_ch_smprintf("%s,csn=%s.000000Z#000000#000#000000",
cookie->cookie_client_signature,
buf);
} else {
cookiestr = slapi_ch_smprintf("%s#%s#%lu",
cookie->cookie_server_signature,
cookie->cookie_client_signature,
cookie->cookie_change_info);
}
}
return (cookiestr);
}
@@ -260,7 +273,12 @@ sync_result_msg(Slapi_PBlock *pb, Sync_Cookie *cookie)
char *cookiestr = sync_cookie2str(cookie);

LDAPControl **ctrl = (LDAPControl **)slapi_ch_calloc(2, sizeof(LDAPControl *));
sync_create_sync_done_control(&ctrl[0], 0, cookiestr);

if (cookie->openldap_compat) {
sync_create_sync_done_control(&ctrl[0], 1, cookiestr);
} else {
sync_create_sync_done_control(&ctrl[0], 0, cookiestr);
}
slapi_pblock_set(pb, SLAPI_RESCONTROLS, ctrl);
slapi_send_ldap_result(pb, 0, NULL, NULL, 0, NULL);

@@ -288,24 +306,39 @@ create_syncinfo_value(int type, const char *cookie, const char **uuids)
return (NULL);
}

/*
* ber_tag_t is an unsigned integer of at least 32 bits
* used to represent a BER tag. It is commonly equivalent
* to a unsigned long.
* ...
* ber_printf(...)
* t
* Tag of the next element. A pointer to a ber_tag_t should be supplied.
*/

ber_tag_t btag = (ber_tag_t)type;

switch (type) {
case LDAP_TAG_SYNC_NEW_COOKIE:
ber_printf(ber, "to", type, cookie);
ber_printf(ber, "to", btag, cookie);
break;
case LDAP_TAG_SYNC_REFRESH_DELETE:
case LDAP_TAG_SYNC_REFRESH_PRESENT:
ber_printf(ber, "t{", type);
if (cookie)
ber_printf(ber, "t{", btag);
if (cookie) {
ber_printf(ber, "s", cookie);
}
/* ber_printf(ber, "b",1); */
ber_printf(ber, "}");
break;
case LDAP_TAG_SYNC_ID_SET:
ber_printf(ber, "t{", type);
if (cookie)
ber_printf(ber, "t{", btag);
if (cookie) {
ber_printf(ber, "s", cookie);
if (uuids)
}
if (uuids) {
ber_printf(ber, "b[v]", 1, uuids);
}
ber_printf(ber, "}");
break;
default:
@@ -471,19 +504,27 @@ sync_cookie_get_change_info(Sync_CallBackData *scbd)
}

Sync_Cookie *
sync_cookie_create(Slapi_PBlock *pb)
sync_cookie_create(Slapi_PBlock *pb, Sync_Cookie *client_cookie)
{

Sync_CallBackData scbd;
int rc;
Sync_CallBackData scbd = {0};
int rc = 0;
Sync_Cookie *sc = (Sync_Cookie *)slapi_ch_calloc(1, sizeof(Sync_Cookie));

scbd.cb_err = SYNC_CALLBACK_PREINIT;
rc = sync_cookie_get_change_info(&scbd);

if (rc == 0) {
sc->cookie_server_signature = sync_cookie_get_server_info(pb);
sc->cookie_client_signature = sync_cookie_get_client_info(pb);
/* If the client is in openldap compat, we need to generate the same. */
if (client_cookie && client_cookie->openldap_compat) {
sc->openldap_compat = client_cookie->openldap_compat;
sc->cookie_client_signature = slapi_ch_strdup(client_cookie->cookie_client_signature);
sc->cookie_server_signature = NULL;
} else {
sc->openldap_compat = false;
sc->cookie_server_signature = sync_cookie_get_server_info(pb);
sc->cookie_client_signature = sync_cookie_get_client_info(pb);
}

if (scbd.cb_err == SYNC_CALLBACK_PREINIT) {
/* changenr is not initialized. */
sc->cookie_change_info = 0;
@@ -513,36 +554,110 @@ sync_cookie_update(Sync_Cookie *sc, Slapi_Entry *ec)
}

Sync_Cookie *
sync_cookie_parse(char *cookie)
sync_cookie_parse(char *cookie, bool *cookie_refresh)
{
char *p, *q;
char *p = NULL;
char *q = NULL;
Sync_Cookie *sc = NULL;

/* This is an rfc compliant initial refresh request */
if (cookie == NULL || *cookie == '\0') {
*cookie_refresh = 1;
return NULL;
}

/*
* Format of cookie: server_signature#client_signature#change_info_number
* If the cookie is malformed, NULL is returned.
*/
/* get ready to parse. */
p = q = cookie;
p = strchr(q, '#');
if (p) {
*p = '\0';
sc = (Sync_Cookie *)slapi_ch_calloc(1, sizeof(Sync_Cookie));
sc->cookie_server_signature = slapi_ch_strdup(q);
q = p + 1;

sc = (Sync_Cookie *)slapi_ch_calloc(1, sizeof(Sync_Cookie));
if (strncmp(cookie, "rid=", 4) == 0) {
/*
* We are in openldap mode.
* The cookies are:
* rid=123,csn=20200525051329.534174Z#000000#000#000000
*/
sc->openldap_compat = true;
p = strchr(q, ',');
if (p == NULL) {
/* No CSN following the rid, must be an init request. */
*cookie_refresh = 1;
/* We need to keep the client rid though */
sc->cookie_client_signature = slapi_ch_strdup(q);
/* server sig and change info do not need to be set. */
sc->cookie_server_signature = NULL;
sc->cookie_change_info = 0;
} else {
/* Ensure that this really is a csn= */
if (strncmp(p, ",csn=", 5) != 0) {
/* Yeah nahhhhhhh */
goto error_return;
}
/* We dont care about the remainder after the . */
if (strlen(p) < 20) {
/* Probably a corrupt CSN. We need at least 20 chars. */
goto error_return;
}
/*
* Replace the , with a '\0' This makes q -> p a str of the rid.
* rid=123,csn=19700101001640.000000Z#000000#000#000000
* ^ ^
* q p
* rid=123\0csn=19700101001640.000000Z#000000#000#000000
*/
PR_ASSERT(p[0] == ',');
p[0] = '\0';
/*
* Now terminate the ulong which is our change num so we can parse it.
* rid=123\0csn=19700101001640.000000Z#000000#000#000000
* ^ ^ ^
* q p[0] p[19]
* rid=123\0csn=19700101001640\0...
*/
PR_ASSERT(p[19] == '.');
p[19] = '\0';
/*
* And move the pointer up to the start of the int we need to parse.
* rid=123\0csn=19700101001640\0...
* ^ ^
* q p +5 -->
* rid=123\0csn=19700101001640\0...
* ^ ^
* q p
*/
p = p + 5;
PR_ASSERT(strlen(p) == 14);
/* We are now ready to parse the csn and create a cookie! */
sc->cookie_client_signature = slapi_ch_strdup(q);
sc->cookie_server_signature = NULL;
/* Get the change number from the string */
sc->cookie_change_info = sync_olcsn2ulong(p);
if (SYNC_INVALID_CHANGENUM == sc->cookie_change_info) {
/* Sad trombone */
goto error_return;
}
/* Done! 🎉 */
}
} else {
/*
* Format of the 389 cookie: server_signature#client_signature#change_info_number
* If the cookie is malformed, NULL is returned.
*/
p = strchr(q, '#');
if (p) {
*p = '\0';
sc->cookie_client_signature = slapi_ch_strdup(q);
sc->cookie_change_info = sync_number2ulong(p + 1);
if (SYNC_INVALID_CHANGENUM == sc->cookie_change_info) {
sc->cookie_server_signature = slapi_ch_strdup(q);
q = p + 1;
p = strchr(q, '#');
if (p) {
*p = '\0';
sc->cookie_client_signature = slapi_ch_strdup(q);
sc->cookie_change_info = sync_number2ulong(p + 1);
if (SYNC_INVALID_CHANGENUM == sc->cookie_change_info) {
goto error_return;
}
} else {
goto error_return;
}
} else {
goto error_return;
}
}
return (sc);
@@ -557,17 +672,30 @@ int
sync_cookie_isvalid(Sync_Cookie *testcookie, Sync_Cookie *refcookie)
{
/* client and server info must match */
if ((testcookie && refcookie) &&
(strcmp(testcookie->cookie_client_signature, refcookie->cookie_client_signature) ||
strcmp(testcookie->cookie_server_signature, refcookie->cookie_server_signature) ||
if (testcookie == NULL || refcookie == NULL) {
return 0;
}
if ((testcookie->openldap_compat != refcookie->openldap_compat ||
strcmp(testcookie->cookie_client_signature, refcookie->cookie_client_signature) ||
testcookie->cookie_change_info == -1 ||
testcookie->cookie_change_info > refcookie->cookie_change_info)) {
return (0);
return 0;
}

if (refcookie->openldap_compat) {
if (testcookie->cookie_server_signature != NULL ||
refcookie->cookie_server_signature != NULL) {
return 0;
}
} else {
if (strcmp(testcookie->cookie_server_signature, refcookie->cookie_server_signature)) {
return 0;
}
}
/* could add an additional check if the requested state in client cookie is still
* available. Accept any state request for now.
*/
return (1);
return 1;
}

void
@@ -701,3 +829,40 @@ sync_number2ulong(char *chgnrstr)
return SYNC_INVALID_CHANGENUM;
}
}

/*
* Why is there a CSN offset?
*
* CSN offset is to bump our csn date to a future time so that
* we always beat openldap in conflicts. I can only hope that
* in 100 years this code is dead, buried, for no one to see
* again. If you are reading this in 2100, William of 2020
* says "I'm so very sorry".
*/

static unsigned long
sync_olcsn2ulong(char *csn) {
struct tm pt = {0};
char *ret = strptime(csn, "%Y%m%d%H%M%S", &pt);
PR_ASSERT(ret);
if (ret == NULL) {
return SYNC_INVALID_CHANGENUM;
}
time_t pepoch = mktime(&pt);
unsigned long px = (unsigned long)pepoch;
PR_ASSERT(px >= CSN_OFFSET);
if (px < CSN_OFFSET) {
return SYNC_INVALID_CHANGENUM;
}
return px - CSN_OFFSET;
}

static void
sync_ulong2olcsn(unsigned long chgnr, char *buf) {
PR_ASSERT(buf);
unsigned long x = chgnr + CSN_OFFSET;
time_t epoch = x;
struct tm t = {0};
localtime_r(&epoch, &t);
strftime(buf, 15, "%Y%m%d%H%M%S", &t);
}

+ 2
- 0
src/lib389/lib389/cli_conf/plugin.py View File

@@ -27,6 +27,7 @@ from lib389.cli_conf.plugins import passthroughauth as cli_passthroughauth
from lib389.cli_conf.plugins import retrochangelog as cli_retrochangelog
from lib389.cli_conf.plugins import automember as cli_automember
from lib389.cli_conf.plugins import posix_winsync as cli_posix_winsync
from lib389.cli_conf.plugins import contentsync as cli_contentsync

SINGULAR = Plugin
MANY = Plugins
@@ -113,6 +114,7 @@ def create_parser(subparsers):
cli_passthroughauth.create_parser(subcommands)
cli_retrochangelog.create_parser(subcommands)
cli_posix_winsync.create_parser(subcommands)
cli_contentsync.create_parser(subcommands)

list_parser = subcommands.add_parser('list', help="List current configured (enabled and disabled) plugins")
list_parser.set_defaults(func=plugin_list)


+ 15
- 0
src/lib389/lib389/cli_conf/plugins/contentsync.py View File

@@ -0,0 +1,15 @@
# --- BEGIN COPYRIGHT BLOCK ---
# Copyright (C) 2020 William Brown <william at blackhats.net.au>
# All rights reserved.
#
# License: GPL (version 3 or any later version).
# See LICENSE for details.
# --- END COPYRIGHT BLOCK ---

from lib389.plugins import ContentSyncPlugin
from lib389.cli_conf import add_generic_plugin_parsers

def create_parser(subparsers):
contentsync_parser = subparsers.add_parser('contentsync', help='Manage and configure Content Sync Plugin (aka syncrepl)')
subcommands = contentsync_parser.add_subparsers(help='action')
add_generic_plugin_parsers(subcommands, ContentSyncPlugin)

+ 13
- 0
src/lib389/lib389/plugins.py View File

@@ -2260,3 +2260,16 @@ class EntryUUIDPlugin(Plugin):
task.create(properties=task_properties)

return task

class ContentSyncPlugin(Plugin):
"""A single instance of Content Sync (aka syncrepl) plugin entry

:param instance: An instance
:type instance: lib389.DirSrv
:param dn: Entry DN
:type dn: str
"""

def __init__(self, instance, dn="cn=Content Synchronization,cn=plugins,cn=config"):
super(ContentSyncPlugin, self).__init__(instance, dn)


Loading…
Cancel
Save