init commit

.gitignore

@@ -10,7 +10,7 @@ __pycache__/
 .ropeproject/
 #django
 local_settings.py
-
+Pipfile
 media/
 !media/keep
 /CACHE/
@@ -43,3 +43,4 @@ secret-key
 # to keep empty dirs
 !.gitkeep
 *.orig
+.vscode/settings.json

INSTALLATION.rst

@@ -10,13 +10,35 @@ Requirements
 
 Install
 =======
+
+.. note::
+    lxml that is one of the dependency of dynamicweb couldn't
+    get build on Python 3.7 so, please use Python 3.5.
+
+
+First install packages from requirements.archlinux.txt or 
+requirements.debian.txt based on your distribution.
+
+
 The quick way:
     ``pip install -r requirements.txt``
 
 Next find the dump.db file on stagging server. Path for the file is under the base application folder.
 
+or you can create one for yourself by running the following commands on dynamicweb server
+
+.. code:: sh
+
+    sudo su - postgres
+    pg_dump app > /tmp/postgres_db.bak
+    exit
+    cp /tmp/postgres_db.bak /root/postgres_db.bak
+
+Now, you can download this using sftp.
+
+
 Install the postgresql server and import the database::
-    ``psql -d app < dump.db``
+    ``psql -d app -U root < dump.db``
     
 **No migration is needed after a clean install, and You are ready to start developing.**
 
@@ -25,9 +47,9 @@ Development
 Project is separated in master branch and development branch, and feature branches.
 Master branch is currently used on `Digital Glarus <https://digitalglarus.ungleich.ch/en-us/digitalglarus/>`_ and `Ungleich blog <https://digitalglarus.ungleich.ch/en-us/blog/>`_.
 
-If You are starting to  create a new feature fork the github `repo <https://github.com/ungleich/dynamicweb>`_ and branch the development branch. 
+If You are starting to create a new feature fork the github `repo <https://github.com/ungleich/dynamicweb>`_ and branch the development branch. 
 
-After You have complited the task create a pull request and ask someone to review the code from other developers. 
+After You have completed the task, create a pull request and ask someone to review the code from other developers. 
 
 **Cheat sheet for branching and forking**:
 

dynamicweb/settings/base.py

@@ -10,7 +10,10 @@ import os
 
 # dotenv
 import dotenv
+import ldap
+
 from django.utils.translation import ugettext_lazy as _
+from django_auth_ldap.config import LDAPSearch, LDAPSearchUnion
 
 logger = logging.getLogger(__name__)
 
@@ -52,7 +55,7 @@ PROJECT_DIR = os.path.abspath(
 )
 
 # load .env file
-dotenv.read_dotenv("{0}/.env".format(PROJECT_DIR))
+dotenv.load_dotenv("{0}/.env".format(PROJECT_DIR))
 
 from multisite import SiteID
 
@@ -240,12 +243,14 @@ DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
         'NAME': 'app',
+        'USER': 'root'
     }
 }
 
 AUTHENTICATION_BACKENDS = (
+    'utils.backend.MyLDAPBackend',
     'guardian.backends.ObjectPermissionBackend',
-    'django.contrib.auth.backends.ModelBackend',
+
 )
 
 # Internationalization
@@ -721,6 +726,35 @@ X_FRAME_OPTIONS = ('SAMEORIGIN' if X_FRAME_OPTIONS_ALLOW_FROM_URI is None else
 
 DEBUG = bool_env('DEBUG')
 
+
+# LDAP setup
+LDAP_SERVER = env('LDAP_SERVER')
+LDAP_ADMIN_DN = env('LDAP_ADMIN_DN')
+LDAP_ADMIN_PASSWORD = env('LDAP_ADMIN_PASSWORD')
+AUTH_LDAP_SERVER = env('LDAPSERVER')
+
+LDAP_CUSTOMER_DN = env('LDAP_CUSTOMER_DN')
+LDAP_CUSTOMER_GROUP_ID = int(env('LDAP_CUSTOMER_GROUP_ID'))
+LDAP_MAX_UID_FILE_PATH = os.environ.get('LDAP_MAX_UID_FILE_PATH',
+                                        os.path.join(os.path.abspath(os.path.dirname(__file__)), 'ldap_max_uid_file')
+)
+LDAP_DEFAULT_START_UID = int(env('LDAP_DEFAULT_START_UID'))
+
+# Search union over OUs
+search_base = env('LDAPSEARCH').split()
+search_base_ldap = [LDAPSearch(x, ldap.SCOPE_SUBTREE, "(uid=%(user)s)") for x in search_base]
+AUTH_LDAP_USER_SEARCH = LDAPSearchUnion(*search_base_ldap)
+AUTH_LDAP_START_TLS = bool(os.environ.get('LDAP_USE_TLS', False))
+
+ENTIRE_SEARCH_BASE = env("ENTIRE_SEARCH_BASE")
+
+
+AUTH_LDAP_USER_ATTR_MAP = {
+    "first_name": "givenName",
+    "last_name": "sn",
+    "email": "mail"
+}
+
 READ_VM_REALM = env('READ_VM_REALM')
 AUTH_NAME = env('AUTH_NAME')
 AUTH_SEED = env('AUTH_SEED')

dynamicweb/settings/ldap_max_uid_file

@@ -0,0 +1 @@
+10173

hosting/views.py

@@ -57,6 +57,8 @@ from utils.hosting_utils import (
 from utils.mailer import BaseEmail
 from utils.stripe_utils import StripeUtils
 from utils.tasks import send_plain_email_task
+from utils.ldap_manager import LdapManager
+
 from utils.views import (
     PasswordResetViewMixin, PasswordResetConfirmViewMixin, LoginViewMixin,
     ResendActivationLinkViewMixin
@@ -394,9 +396,12 @@ class PasswordResetConfirmView(HostingContextMixin,
         if user is not None and default_token_generator.check_token(user,
                                                                     token):
             if form.is_valid():
+                ldap_manager = LdapManager()
                 new_password = form.cleaned_data['new_password2']
+                user.create_ldap_account()
                 user.set_password(new_password)
                 user.save()
+                ldap_manager.change_password(user.username, user.password)
                 messages.success(request, _('Password has been reset.'))
 
                 # Change opennebula password

membership/migrations/0011_customuser_username.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2019-12-10 10:52
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('membership', '0010_customuser_import_stripe_bill_remark'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='customuser',
+            name='username',
+            field=models.CharField(max_length=50, null=True),
+        ),
+    ]

membership/migrations/0012_auto_20191210_1141.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2019-12-10 11:41
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('membership', '0011_customuser_username'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='customuser',
+            name='username',
+            field=models.CharField(max_length=50, null=True, unique=True),
+        ),
+    ]

membership/migrations/0013_customuser_in_ldap.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2019-12-10 15:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('membership', '0012_auto_20191210_1141'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='customuser',
+            name='in_ldap',
+            field=models.BooleanField(default=False),
+        ),
+    ]

membership/migrations/0014_remove_customuser_in_ldap.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2019-12-10 15:36
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('membership', '0013_customuser_in_ldap'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='customuser',
+            name='in_ldap',
+        ),
+    ]

membership/migrations/0015_customuser_in_ldap.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2019-12-10 17:05
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('membership', '0014_remove_customuser_in_ldap'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='customuser',
+            name='in_ldap',
+            field=models.BooleanField(default=False),
+        ),
+    ]

membership/models.py

@@ -1,5 +1,6 @@
-from datetime import datetime
+import logging
 
+from datetime import datetime
 from django.conf import settings
 from django.contrib.auth.hashers import make_password
 from django.contrib.auth.models import AbstractBaseUser, BaseUserManager, \
@@ -7,13 +8,16 @@ from django.contrib.auth.models import AbstractBaseUser, BaseUserManager, \
 from django.contrib.sites.models import Site
 from django.core.urlresolvers import reverse
 from django.core.validators import RegexValidator
-from django.db import models
+from django.db import models, IntegrityError
 from django.utils.crypto import get_random_string
 from django.utils.translation import ugettext_lazy as _
 
 from utils.mailer import BaseEmail
 from utils.mailer import DigitalGlarusRegistrationMailer
 from utils.stripe_utils import StripeUtils
+from utils.ldap_manager import LdapManager
+
+logger = logging.getLogger(__name__)
 
 REGISTRATION_MESSAGE = {'subject': "Validation mail",
                         'message': 'Please validate Your account under this link '
@@ -42,6 +46,7 @@ class MyUserManager(BaseUserManager):
         user.is_admin = False
         user.set_password(password)
         user.save(using=self._db)
+        user.create_ldap_account()
         return user
 
     def create_superuser(self, email, name, password):
@@ -63,13 +68,43 @@ def get_validation_slug():
     return make_password(None)
 
 
+def get_first_and_last_name(full_name):
+    first_name, *last_name = full_name.split(" ")
+    first_name = first_name
+    last_name = " ".join(last_name)
+    return first_name, last_name
+
+
+def assign_username(user):
+    if not user.username:
+        first_name, last_name = get_first_and_last_name(user.name)
+        user.username = first_name.lower() + last_name.lower()
+        user.username = "".join(user.username.split())
+        try:
+            user.save()
+        except IntegrityError:
+            try:
+                user.username = user.username + str(user.id)
+                user.save()
+            except IntegrityError:
+                while True:
+                    user.username = user.username + str(random.randint(0, 2 ** 50))
+                    try:
+                        user.save()
+                    except IntegrityError:
+                        continue
+                    else:
+                        break
+
+
 class CustomUser(AbstractBaseUser, PermissionsMixin):
     VALIDATED_CHOICES = ((0, 'Not validated'), (1, 'Validated'))
     site = models.ForeignKey(Site, default=1)
     name = models.CharField(max_length=50)
     email = models.EmailField(unique=True)
-
+    username = models.CharField(max_length=50, unique=True, null=True)
     validated = models.IntegerField(choices=VALIDATED_CHOICES, default=0)
+    in_ldap = models.BooleanField(default=False)
     # By default, we initialize the validation_slug with appropriate value
     # This is required for User(page) admin
     validation_slug = models.CharField(
@@ -164,6 +199,34 @@ class CustomUser(AbstractBaseUser, PermissionsMixin):
         # The user is identified by their email address
         return self.email
 
+    def create_ldap_account(self):
+        # create ldap account for user if it does not exists already.
+        if self.in_ldap:
+            return
+
+        assign_username(self)
+        ldap_manager = LdapManager()
+        try:
+            user_exists_in_ldap, entries = ldap_manager.check_user_exists(
+                uid=self.username,
+                attributes=['uid', 'givenName', 'sn', 'mail', 'userPassword'],
+                search_base=settings.ENTIRE_SEARCH_BASE,
+                search_attr='uid'
+            )
+        except Exception:
+            logger.exception("Exception occur while searching for user in LDAP")
+        else:
+            if not user_exists_in_ldap:
+                # IF no ldap account
+                first_name, last_name = get_first_and_last_name(self.name)
+                if not last_name:
+                    last_name = first_name
+
+                ldap_manager.create_user(self.username, password=self.password,
+                                         firstname=first_name, lastname=last_name,
+                                         email=self.email)
+                self.in_ldap = True
+                self.save()
     def __str__(self):  # __unicode__ on Python 2
         return self.email
 

requirements.archlinux.txt

@@ -1 +1,2 @@
+base-devel
 libmemcached

utils/backend.py

@@ -0,0 +1,73 @@
+
+import logging
+
+from membership.models import CustomUser
+logger = logging.getLogger(__name__)
+
+class MyLDAPBackend(object):
+    def authenticate(self, email, password):
+        try:
+            user = CustomUser.objects.get(email=email)
+        except CustomUser.DoesNotExist:
+            # User does not exists in Database
+            return None
+        else:
+            user.create_ldap_account()
+            if user.check_password(password):
+                return user
+            else:
+                return None
+
+            # # User exists in Database
+            # user.create_ldap_account()
+            # # User does not have a username
+            # if not user.username:
+            #     assign_username(user)
+            #
+            # ldap_manager = LdapManager()
+            # try:
+            #     user_exists_in_ldap, entries = ldap_manager.check_user_exists(
+            #         uid=user.username,
+            #         attributes=['uid', 'givenName', 'sn', 'mail', 'userPassword'],
+            #         search_base=settings.ENTIRE_SEARCH_BASE,
+            #         search_attr='uid'
+            #     )
+            # except Exception:
+            #     logger.exception("Exception occur while searching for user in LDAP")
+            # else:
+            #     ph = PasswordHasher()
+            #     if user_exists_in_ldap:
+            #         # User Exists in LDAP
+            #         password_hash_from_ldap = entries[0]["userPassword"].value
+            #         try:
+            #             ph.verify(password_hash_from_ldap, password)
+            #         except Exception:
+            #             # Incorrect LDAP Password
+            #             return None
+            #         else:
+            #             # Correct LDAP Password
+            #             return user
+            #     else:
+            #         # User does not exists in LDAP
+            #         if user.check_password(password):
+            #             # Password is correct as per database
+            #             first_name, last_name = get_first_and_last_name(user.name)
+            #             if not last_name:
+            #                 last_name = first_name
+            #
+            #             ldap_manager.create_user(user.username, password=ph.hash(password),
+            #                                      firstname=first_name, lastname=last_name,
+            #                                      email=user.email)
+            #             user.password = "IN_LDAP"
+            #             user.save()
+            #             return user
+            #         else:
+            #             # Incorrect Password
+            #             print("Incorrect password")
+            #             return None
+
+    def get_user(self, user_id):
+        try:
+            return CustomUser.objects.get(pk=user_id)
+        except CustomUser.DoesNotExist:
+            return None

utils/ldap_manager.py

@@ -0,0 +1,279 @@
+import base64
+import hashlib
+import random
+import ldap3
+import logging
+
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
+
+
+class LdapManager:
+    __instance = None
+
+    def __new__(cls):
+        if LdapManager.__instance is None:
+            LdapManager.__instance = object.__new__(cls)
+        return LdapManager.__instance
+
+    def __init__(self):
+        """
+        Initialize the LDAP subsystem.
+        """
+        self.rng = random.SystemRandom()
+        self.server = ldap3.Server(settings.AUTH_LDAP_SERVER)
+
+    def get_admin_conn(self):
+        """
+        Return a bound :class:`ldap3.Connection` instance which has write
+        permissions on the dn in which the user accounts reside.
+        """
+        conn = self.get_conn(user=settings.LDAP_ADMIN_DN,
+                             password=settings.LDAP_ADMIN_PASSWORD,
+                             raise_exceptions=True)
+        conn.bind()
+        return conn
+
+    def get_conn(self, **kwargs):
+        """
+        Return an unbound :class:`ldap3.Connection` which talks to the configured
+        LDAP server.
+
+        The *kwargs* are passed to the constructor of :class:`ldap3.Connection` and
+        can be used to set *user*, *password* and other useful arguments.
+        """
+        return ldap3.Connection(self.server, **kwargs)
+
+    def _ssha_password(self, password):
+        """
+        Apply the SSHA password hashing scheme to the given *password*.
+        *password* must be a :class:`bytes` object, containing the utf-8
+        encoded password.
+
+        Return a :class:`bytes` object containing ``ascii``-compatible data
+        which can be used as LDAP value, e.g. after armoring it once more using
+        base64 or decoding it to unicode from ``ascii``.
+        """
+        SALT_BYTES = 15
+
+        sha1 = hashlib.sha1()
+        salt = self.rng.getrandbits(SALT_BYTES * 8).to_bytes(SALT_BYTES,
+                                                               "little")
+        sha1.update(password)
+        sha1.update(salt)
+
+        digest = sha1.digest()
+        passwd = b"{SSHA}" + base64.b64encode(digest + salt)
+        return passwd
+
+    def create_user(self, user, password, firstname, lastname, email):
+        conn = self.get_admin_conn()
+        uidNumber = self._get_max_uid() + 1
+        logger.debug("uidNumber={uidNumber}".format(uidNumber=uidNumber))
+        user_exists = True
+        while user_exists:
+            user_exists, _ = self.check_user_exists(
+                "",
+                '(&(objectClass=inetOrgPerson)(objectClass=posixAccount)'
+                '(objectClass=top)(uidNumber={uidNumber}))'.format(
+                    uidNumber=uidNumber
+                )
+            )
+            if user_exists:
+                logger.debug(
+                    "{uid} exists. Trying next.".format(uid=uidNumber)
+                )
+                uidNumber += 1
+        logger.debug("{uid} does not exist. Using it".format(uid=uidNumber))
+        self._set_max_uid(uidNumber)
+        try:
+            uid = user
+            conn.add("uid={uid},{customer_dn}".format(
+                    uid=uid, customer_dn=settings.LDAP_CUSTOMER_DN
+                ),
+                ["inetOrgPerson", "posixAccount", "ldapPublickey"],
+                {
+                    "uid": [uid],
+                    "sn": [lastname],
+                    "givenName": [firstname],
+                    "cn": [uid],
+                    "displayName": ["{} {}".format(firstname, lastname)],
+                    "uidNumber": [str(uidNumber)],
+                    "gidNumber": [str(settings.LDAP_CUSTOMER_GROUP_ID)],
+                    "loginShell": ["/bin/bash"],
+                    "homeDirectory": ["/home/{}".format(user)],
+                    "mail": email,
+                    "userPassword": [password]
+                }
+            )
+            logger.debug('Created user %s %s' % (user.encode('utf-8'),
+                                                 uidNumber))
+        except Exception as ex:
+            logger.debug('Could not create user %s' % user.encode('utf-8'))
+            logger.error("Exception: " + str(ex))
+            raise Exception(ex)
+        finally:
+            conn.unbind()
+
+    def change_password(self, uid, new_password):
+        """
+        Changes the password of the user identified by user_dn
+
+        :param uid: str The uid that identifies the user
+        :param new_password: str The new password string
+        :return: True if password was changed successfully False otherwise
+        """
+        conn = self.get_admin_conn()
+
+        # Make sure the user exists first to change his/her details
+        user_exists, entries = self.check_user_exists(
+            uid=uid,
+            search_base=settings.ENTIRE_SEARCH_BASE
+        )
+        return_val = False
+        if user_exists:
+            try:
+                return_val = conn.modify(
+                    entries[0].entry_dn,
+                    {
+                        "userpassword": (
+                            ldap3.MODIFY_REPLACE,
+                            [new_password]
+                        )
+                    }
+                )
+            except Exception as ex:
+                logger.error("Exception: " + str(ex))
+        else:
+            logger.error("User {} not found".format(uid))
+
+        conn.unbind()
+        return return_val
+
+    def change_user_details(self, uid, details):
+        """
+        Updates the user details as per given values in kwargs of the user
+        identified by user_dn.
+
+        Assumes that all attributes passed in kwargs are valid.
+
+        :param uid: str The uid that identifies the user
+        :param details: dict A dictionary containing the new values
+        :return: True if user details were updated successfully False otherwise
+        """
+        conn = self.get_admin_conn()
+
+        # Make sure the user exists first to change his/her details
+        user_exists, entries = self.check_user_exists(
+            uid=uid,
+            search_base=settings.ENTIRE_SEARCH_BASE
+        )
+
+        return_val = False
+        if user_exists:
+            details_dict = {k: (ldap3.MODIFY_REPLACE, [v.encode("utf-8")]) for
+                            k, v in details.items()}
+            try:
+                return_val = conn.modify(entries[0].entry_dn, details_dict)
+                msg = "success"
+            except Exception as ex:
+                msg = str(ex)
+                logger.error("Exception: " + msg)
+            finally:
+                conn.unbind()
+        else:
+            msg = "User {} not found".format(uid)
+            logger.error(msg)
+            conn.unbind()
+        return return_val, msg
+
+    def check_user_exists(self, uid, search_filter="", attributes=None,
+                          search_base=settings.LDAP_CUSTOMER_DN, search_attr="uid"):
+        """
+        Check if the user with the given uid exists in the customer group.
+
+        :param uid: str representing the user
+        :param search_filter: str representing the filter condition to find
+                            users. If its empty, the search finds the user with
+                            the given uid.
+        :param attributes: list A list of str representing all the attributes
+                           to be obtained in the result entries
+        :param search_base: str
+        :return: tuple (bool, [ldap3.abstract.entry.Entry ..])
+                       A bool indicating if the user exists
+                       A list of all entries obtained in the search
+        """
+        conn = self.get_admin_conn()
+        entries = []
+        try:
+            result = conn.search(
+                search_base=search_base,
+                search_filter=search_filter if len(search_filter) > 0 else
+                '(uid={uid})'.format(uid=uid),
+                attributes=attributes
+            )
+            entries = conn.entries
+        finally:
+            conn.unbind()
+        return result, entries
+
+    def delete_user(self, uid):
+        """
+        Deletes the user with the given uid from ldap
+
+        :param uid: str representing the user
+        :return: True if the delete was successful False otherwise
+        """
+        conn = self.get_admin_conn()
+        try:
+            return_val = conn.delete(
+                ("uid={uid}," + settings.LDAP_CUSTOMER_DN).format(uid=uid),
+            )
+            msg = "success"
+        except Exception as ex:
+            msg = str(ex)
+            logger.error("Exception: " + msg)
+            return_val = False
+        finally:
+            conn.unbind()
+        return return_val, msg
+
+    def _set_max_uid(self, max_uid):
+        """
+        a utility function to save max_uid value to a file
+
+        :param max_uid: an integer representing the max uid
+        :return:
+        """
+        with open(settings.LDAP_MAX_UID_FILE_PATH, 'w+') as handler:
+            handler.write(str(max_uid))
+
+    def _get_max_uid(self):
+        """
+        A utility function to read the max uid value that was previously set
+
+        :return: An integer representing the max uid value that was previously
+         set
+        """
+        try:
+            with open(settings.LDAP_MAX_UID_FILE_PATH, 'r+') as handler:
+                try:
+                    return_value = int(handler.read())
+                except ValueError as ve:
+                    logger.error(
+                        "Error reading int value from {}. {}"
+                        "Returning default value {} instead".format(
+                            settings.LDAP_MAX_UID_PATH,
+                            str(ve),
+                            settings.LDAP_DEFAULT_START_UID
+                        )
+                    )
+                    return_value = settings.LDAP_DEFAULT_START_UID
+                return return_value
+        except FileNotFoundError as fnfe:
+            logger.error("File not found : " + str(fnfe))
+            return_value = settings.LDAP_DEFAULT_START_UID
+            logger.error("So, returning UID={}".format(return_value))
+            return return_value
+