# Copyright (c) 2016 Red Hat, Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from unittest import mock
import uuid

import ddt
from oslo_config import cfg
from oslo_utils import timeutils

from cinder.common import constants
from cinder import exception
from cinder import objects
from cinder.objects import fields
from cinder.tests.unit import fake_constants as fake
from cinder.tests.unit import fake_service
from cinder.tests.unit import utils
from cinder.tests.unit import volume as base
import cinder.volume
from cinder.volume import manager
from cinder.volume import rpcapi as volume_rpcapi
from cinder.volume import volume_utils


CONF = cfg.CONF


@ddt.ddt
class ReplicationTestCase(base.BaseVolumeTestCase):
    def setUp(self):
        super(ReplicationTestCase, self).setUp()
        self.host = 'host@backend#pool'
        self.manager = manager.VolumeManager(host=self.host)

    @mock.patch('cinder.objects.VolumeList.get_all')
    @mock.patch('cinder.volume.driver.BaseVD.failover_host',
                side_effect=exception.InvalidReplicationTarget(''))
    @ddt.data(('backend2', 'default', fields.ReplicationStatus.FAILED_OVER),
              ('backend2', 'backend3', fields.ReplicationStatus.FAILED_OVER),
              (None, 'backend2', fields.ReplicationStatus.ENABLED),
              ('', 'backend2', fields.ReplicationStatus.ENABLED))
    @ddt.unpack
    def test_failover_host_invalid_target(self, svc_backend, new_backend,
                                          expected, mock_failover,
                                          mock_getall):
        """Test replication failover_host with invalid_target.

        When failingover fails due to an invalid target exception we return
        replication_status to its previous status, and we decide what that is
        depending on the currect active backend.
        """
        svc = utils.create_service(
            self.context,
            {'host': self.host,
             'binary': constants.VOLUME_BINARY,
             'active_backend_id': svc_backend,
             'replication_status': fields.ReplicationStatus.FAILING_OVER})

        self.manager.failover_host(self.context, new_backend)
        mock_getall.assert_called_once_with(self.context,
                                            filters={'host': self.host},
                                            limit=None, offset=None)
        mock_failover.assert_called_once_with(self.context,
                                              [],
                                              secondary_id=new_backend,
                                              groups=[])

        db_svc = objects.Service.get_by_id(self.context, svc.id)
        self.assertEqual(expected, db_svc.replication_status)

    @mock.patch('cinder.volume.driver.BaseVD.failover_host',
                mock.Mock(side_effect=exception.VolumeDriverException('')))
    def test_failover_host_driver_exception(self):
        svc = utils.create_service(
            self.context,
            {'host': self.host,
             'binary': constants.VOLUME_BINARY,
             'active_backend_id': None,
             'replication_status': fields.ReplicationStatus.FAILING_OVER})

        self.manager.failover_host(self.context, mock.sentinel.backend_id)

        db_svc = objects.Service.get_by_id(self.context, svc.id)
        self.assertEqual(fields.ReplicationStatus.FAILOVER_ERROR,
                         db_svc.replication_status)

    @mock.patch('cinder.objects.Service.is_up', True)
    @mock.patch.object(volume_rpcapi.VolumeAPI, 'failover')
    @mock.patch.object(cinder.db, 'conditional_update')
    @mock.patch.object(objects.ServiceList, 'get_all')
    def test_failover(self, mock_get_all, mock_db_update, mock_failover):
        """Test replication failover."""

        service = fake_service.fake_service_obj(self.context,
                                                binary='cinder-volume')
        mock_get_all.return_value = [service]
        mock_db_update.return_value = {'replication_status': 'enabled'}
        volume_api = cinder.volume.api.API()
        volume_api.failover(self.context, host=CONF.host, cluster_name=None)
        mock_failover.assert_called_once_with(self.context, service, None)

    @mock.patch.object(volume_rpcapi.VolumeAPI, 'failover')
    @mock.patch.object(cinder.db, 'conditional_update')
    @mock.patch.object(cinder.db, 'service_get_all')
    def test_failover_unexpected_status(self, mock_db_get_all, mock_db_update,
                                        mock_failover):
        """Test replication failover unexpected status."""

        mock_db_get_all.return_value = [fake_service.fake_service_obj(
            self.context,
            binary=constants.VOLUME_BINARY)]
        mock_db_update.return_value = None
        volume_api = cinder.volume.api.API()
        self.assertRaises(exception.InvalidInput,
                          volume_api.failover,
                          self.context,
                          host=CONF.host,
                          cluster_name=None)

    @mock.patch.object(volume_rpcapi.VolumeAPI, 'freeze_host')
    @mock.patch.object(cinder.db, 'conditional_update', return_value=1)
    @mock.patch.object(cinder.objects.ServiceList, 'get_all')
    def test_freeze_host(self, mock_get_all, mock_db_update,
                         mock_freeze):
        """Test replication freeze_host."""

        service = fake_service.fake_service_obj(self.context,
                                                binary=constants.VOLUME_BINARY)
        mock_get_all.return_value = [service]
        mock_freeze.return_value = True
        volume_api = cinder.volume.api.API()
        volume_api.freeze_host(self.context, host=CONF.host, cluster_name=None)
        mock_freeze.assert_called_once_with(self.context, service)

    @mock.patch.object(volume_rpcapi.VolumeAPI, 'freeze_host')
    @mock.patch.object(cinder.db, 'conditional_update')
    @mock.patch.object(cinder.db, 'service_get_all')
    def test_freeze_host_unexpected_status(self, mock_get_all,
                                           mock_db_update,
                                           mock_freeze):
        """Test replication freeze_host unexpected status."""

        mock_get_all.return_value = [fake_service.fake_service_obj(
            self.context,
            binary=constants.VOLUME_BINARY)]
        mock_db_update.return_value = None
        volume_api = cinder.volume.api.API()
        self.assertRaises(exception.InvalidInput,
                          volume_api.freeze_host,
                          self.context,
                          host=CONF.host,
                          cluster_name=None)

    @mock.patch.object(volume_rpcapi.VolumeAPI, 'thaw_host')
    @mock.patch.object(cinder.db, 'conditional_update', return_value=1)
    @mock.patch.object(cinder.objects.ServiceList, 'get_all')
    def test_thaw_host(self, mock_get_all, mock_db_update,
                       mock_thaw):
        """Test replication thaw_host."""

        service = fake_service.fake_service_obj(self.context,
                                                binary=constants.VOLUME_BINARY)
        mock_get_all.return_value = [service]
        mock_thaw.return_value = True
        volume_api = cinder.volume.api.API()
        volume_api.thaw_host(self.context, host=CONF.host, cluster_name=None)
        mock_thaw.assert_called_once_with(self.context, service)

    @mock.patch.object(volume_rpcapi.VolumeAPI, 'thaw_host')
    @mock.patch.object(cinder.db, 'conditional_update')
    @mock.patch.object(cinder.db, 'service_get_all')
    def test_thaw_host_unexpected_status(self, mock_get_all,
                                         mock_db_update,
                                         mock_thaw):
        """Test replication thaw_host unexpected status."""

        mock_get_all.return_value = [fake_service.fake_service_obj(
            self.context,
            binary=constants.VOLUME_BINARY)]
        mock_db_update.return_value = None
        volume_api = cinder.volume.api.API()
        self.assertRaises(exception.InvalidInput,
                          volume_api.thaw_host,
                          self.context,
                          host=CONF.host, cluster_name=None)

    @mock.patch('cinder.volume.driver.BaseVD.failover_completed')
    def test_failover_completed(self, completed_mock):
        rep_field = fields.ReplicationStatus
        svc = objects.Service(self.context, host=self.volume.host,
                              binary=constants.VOLUME_BINARY,
                              replication_status=rep_field.ENABLED)
        svc.create()
        self.volume.failover_completed(
            self.context,
            {'active_backend_id': 'secondary',
             'replication_status': rep_field.FAILED_OVER})
        service = objects.Service.get_by_id(self.context, svc.id)
        self.assertEqual('secondary', service.active_backend_id)
        self.assertEqual('failed-over', service.replication_status)
        completed_mock.assert_called_once_with(self.context, 'secondary')

    @mock.patch('cinder.volume.driver.BaseVD.failover_completed', wraps=True)
    def test_failover_completed_driver_failure(self, completed_mock):
        rep_field = fields.ReplicationStatus
        svc = objects.Service(self.context, host=self.volume.host,
                              binary=constants.VOLUME_BINARY,
                              replication_status=rep_field.ENABLED)
        svc.create()
        self.volume.failover_completed(
            self.context,
            {'active_backend_id': 'secondary',
             'replication_status': rep_field.FAILED_OVER})
        service = objects.Service.get_by_id(self.context, svc.id)
        self.assertEqual('secondary', service.active_backend_id)
        self.assertEqual(rep_field.ERROR, service.replication_status)
        self.assertTrue(service.disabled)
        self.assertIsNotNone(service.disabled_reason)
        completed_mock.assert_called_once_with(self.context, 'secondary')

    @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover_completed')
    def test_finish_failover_non_clustered(self, completed_mock):
        svc = mock.Mock(is_clustered=None)
        self.volume.finish_failover(self.context, svc, mock.sentinel.updates)
        svc.update.assert_called_once_with(mock.sentinel.updates)
        svc.save.assert_called_once_with()
        completed_mock.assert_not_called()

    @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover_completed')
    def test_finish_failover_clustered(self, completed_mock):
        svc = mock.Mock(cluster_name='cluster_name')
        updates = {'status': 'error'}
        self.volume.finish_failover(self.context, svc, updates)
        completed_mock.assert_called_once_with(self.context, svc, updates)
        svc.cluster.status = 'error'
        svc.cluster.save.assert_called_once()

    @ddt.data(None, 'cluster_name')
    @mock.patch('cinder.volume.manager.VolumeManager.finish_failover')
    @mock.patch('cinder.volume.manager.VolumeManager._get_my_volumes')
    def test_failover_manager(self, cluster, get_vols_mock, finish_mock):
        """Test manager's failover method for clustered and not clustered."""
        rep_field = fields.ReplicationStatus
        svc = objects.Service(self.context, host=self.volume.host,
                              binary=constants.VOLUME_BINARY,
                              cluster_name=cluster,
                              replication_status=rep_field.ENABLED)
        svc.create()

        vol = objects.Volume(self.context, host=self.volume.host)
        vol.create()

        get_vols_mock.return_value = [vol]

        with mock.patch.object(self.volume, 'driver') as driver:
            called, not_called = driver.failover_host, driver.failover
            if cluster:
                called, not_called = not_called, called

            called.return_value = ('secondary', [{'volume_id': vol.id,
                                   'updates': {'status': 'error'}}], [])

            self.volume.failover(self.context,
                                 secondary_backend_id='secondary')

        not_called.assert_not_called()
        called.assert_called_once_with(self.context, [vol],
                                       secondary_id='secondary', groups=[])

        expected_update = {'replication_status': rep_field.FAILED_OVER,
                           'active_backend_id': 'secondary',
                           'disabled': True,
                           'disabled_reason': 'failed-over'}
        finish_mock.assert_called_once_with(self.context, svc, expected_update)

        volume = objects.Volume.get_by_id(self.context, vol.id)
        self.assertEqual('error', volume.status)

    @ddt.data(('host1', None), (None, 'mycluster'))
    @ddt.unpack
    def test_failover_api_fail_multiple_results(self, host, cluster):
        """Fail if we try to failover multiple backends in the same request."""
        rep_field = fields.ReplicationStatus
        clusters = [
            objects.Cluster(self.context,
                            name='mycluster@backend1',
                            replication_status=rep_field.ENABLED,
                            binary=constants.VOLUME_BINARY),
            objects.Cluster(self.context,
                            name='mycluster@backend2',
                            replication_status=rep_field.ENABLED,
                            binary=constants.VOLUME_BINARY)
        ]
        clusters[0].create()
        clusters[1].create()
        services = [
            objects.Service(self.context, host='host1@backend1',
                            cluster_name=clusters[0].name,
                            replication_status=rep_field.ENABLED,
                            binary=constants.VOLUME_BINARY),
            objects.Service(self.context, host='host1@backend2',
                            cluster_name=clusters[1].name,
                            replication_status=rep_field.ENABLED,
                            binary=constants.VOLUME_BINARY),
        ]
        services[0].create()
        services[1].create()
        self.assertRaises(exception.Invalid,
                          self.volume_api.failover, self.context, host,
                          cluster)

    def test_failover_api_not_found(self):
        self.assertRaises(exception.ServiceNotFound, self.volume_api.failover,
                          self.context, 'host1', None)

    @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover')
    def test_failover_api_success_multiple_results(self, failover_mock):
        """Succeed to failover multiple services for the same backend."""
        rep_field = fields.ReplicationStatus
        cluster_name = 'mycluster@backend1'
        cluster = objects.Cluster(self.context,
                                  name=cluster_name,
                                  replication_status=rep_field.ENABLED,
                                  binary=constants.VOLUME_BINARY)
        cluster.create()
        services = [
            objects.Service(self.context, host='host1@backend1',
                            cluster_name=cluster_name,
                            replication_status=rep_field.ENABLED,
                            binary=constants.VOLUME_BINARY),
            objects.Service(self.context, host='host2@backend1',
                            cluster_name=cluster_name,
                            replication_status=rep_field.ENABLED,
                            binary=constants.VOLUME_BINARY),
        ]
        services[0].create()
        services[1].create()

        self.volume_api.failover(self.context, None, cluster_name,
                                 mock.sentinel.secondary_id)

        for service in services + [cluster]:
            self.assertEqual(rep_field.ENABLED, service.replication_status)
            service.refresh()
            self.assertEqual(rep_field.FAILING_OVER,
                             service.replication_status)

        failover_mock.assert_called_once_with(self.context, mock.ANY,
                                              mock.sentinel.secondary_id)
        self.assertEqual(services[0].id, failover_mock.call_args[0][1].id)

    @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover')
    def test_failover_api_success_multiple_results_not_updated(self,
                                                               failover_mock):
        """Succeed to failover even if a service is not updated."""
        rep_field = fields.ReplicationStatus
        cluster_name = 'mycluster@backend1'
        cluster = objects.Cluster(self.context,
                                  name=cluster_name,
                                  replication_status=rep_field.ENABLED,
                                  binary=constants.VOLUME_BINARY)
        cluster.create()
        services = [
            objects.Service(self.context, host='host1@backend1',
                            cluster_name=cluster_name,
                            replication_status=rep_field.ENABLED,
                            binary=constants.VOLUME_BINARY),
            objects.Service(self.context, host='host2@backend1',
                            cluster_name=cluster_name,
                            replication_status=rep_field.ERROR,
                            binary=constants.VOLUME_BINARY),
        ]
        services[0].create()
        services[1].create()

        self.volume_api.failover(self.context, None, cluster_name,
                                 mock.sentinel.secondary_id)

        for service in services[:1] + [cluster]:
            service.refresh()
            self.assertEqual(rep_field.FAILING_OVER,
                             service.replication_status)

        services[1].refresh()
        self.assertEqual(rep_field.ERROR, services[1].replication_status)

        failover_mock.assert_called_once_with(self.context, mock.ANY,
                                              mock.sentinel.secondary_id)
        self.assertEqual(services[0].id, failover_mock.call_args[0][1].id)

    @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover')
    def test_failover_api_fail_multiple_results_not_updated(self,
                                                            failover_mock):
        """Fail if none of the services could be updated."""
        rep_field = fields.ReplicationStatus
        cluster_name = 'mycluster@backend1'
        cluster = objects.Cluster(self.context,
                                  name=cluster_name,
                                  replication_status=rep_field.ENABLED,
                                  binary=constants.VOLUME_BINARY)
        cluster.create()
        down_time = timeutils.datetime.datetime(1970, 1, 1)
        services = [
            # This service is down
            objects.Service(self.context, host='host1@backend1',
                            cluster_name=cluster_name,
                            replication_status=rep_field.ENABLED,
                            created_at=down_time,
                            updated_at=down_time,
                            modified_at=down_time,
                            binary=constants.VOLUME_BINARY),
            # This service is not with the right replication status
            objects.Service(self.context, host='host2@backend1',
                            cluster_name=cluster_name,
                            replication_status=rep_field.ERROR,
                            binary=constants.VOLUME_BINARY),
        ]
        services[0].create()
        services[1].create()

        self.assertRaises(exception.InvalidInput,
                          self.volume_api.failover, self.context, None,
                          cluster_name, mock.sentinel.secondary_id)

        for service in services:
            svc = objects.Service.get_by_id(self.context, service.id)
            self.assertEqual(service.replication_status,
                             svc.replication_status)

        cluster.refresh()
        self.assertEqual(rep_field.ENABLED, cluster.replication_status)

        failover_mock.assert_not_called()

    def _check_failover_db(self, get_method, expected_results):
        db_data = get_method.get_all(self.context, None)
        db_data = {e.id: e for e in db_data}
        for expected in expected_results:
            id_ = expected['id']
            for key, value in expected.items():
                self.assertEqual(value, getattr(db_data[id_], key),
                                 '(%s) ref=%s != act=%s' % (
                                     key, expected, dict(db_data[id_])))

    def _test_failover_model_updates(self, in_volumes, in_snapshots,
                                     driver_volumes, driver_result,
                                     out_volumes, out_snapshots,
                                     in_groups=None, out_groups=None,
                                     driver_group_result=None,
                                     secondary_id=None):
        host = volume_utils.extract_host(self.manager.host)
        utils.create_service(self.context, {'host': host,
                                            'binary': constants.VOLUME_BINARY})
        for volume in in_volumes:
            utils.create_volume(self.context, self.manager.host, **volume)

        for snapshot in in_snapshots:
            utils.create_snapshot(self.context, **snapshot)

        for group in in_groups:
            utils.create_group(self.context, self.manager.host, **group)

        with mock.patch.object(
                self.manager.driver, 'failover_host',
                return_value=(secondary_id, driver_result,
                              driver_group_result)) as driver_mock:
            self.manager.failover_host(self.context, secondary_id)

            self.assertSetEqual(driver_volumes,
                                {v.id for v in driver_mock.call_args[0][1]})

        self._check_failover_db(objects.VolumeList, out_volumes)
        self._check_failover_db(objects.SnapshotList, out_snapshots)
        self._check_failover_db(objects.GroupList, out_groups)

    @mock.patch('cinder.volume.volume_utils.is_group_a_type')
    def test_failover_host_model_updates(self, mock_group_type):
        status = fields.ReplicationStatus
        mock_group_type.return_value = True
        in_groups = [
            {'id': str(uuid.uuid4()), 'status': 'available',
             'group_type_id': fake.GROUP_TYPE_ID,
             'volume_type_ids': [fake.VOLUME_TYPE_ID],
             'replication_status': status.FAILOVER_ERROR},
            {'id': str(uuid.uuid4()), 'status': 'available',
             'group_type_id': fake.GROUP_TYPE_ID,
             'volume_type_ids': [fake.VOLUME_TYPE_ID],
             'replication_status': status.ENABLED},
        ]
        driver_group_result = [
            {'group_id': in_groups[0]['id'],
             'updates': {'replication_status': status.FAILOVER_ERROR}},
            {'group_id': in_groups[1]['id'],
             'updates': {'replication_status': status.FAILED_OVER}},
        ]
        out_groups = [
            {'id': in_groups[0]['id'], 'status': 'error',
             'replication_status': status.FAILOVER_ERROR},
            {'id': in_groups[1]['id'], 'status': in_groups[1]['status'],
             'replication_status': status.FAILED_OVER},
        ]

        # test volumes
        in_volumes = [
            {'id': str(uuid.uuid4()), 'status': 'available',
             'replication_status': status.DISABLED},
            {'id': str(uuid.uuid4()), 'status': 'in-use',
             'replication_status': status.NOT_CAPABLE},
            {'id': str(uuid.uuid4()), 'status': 'available',
             'replication_status': status.FAILOVER_ERROR},
            {'id': str(uuid.uuid4()), 'status': 'in-use',
             'replication_status': status.ENABLED},
            {'id': str(uuid.uuid4()), 'status': 'available',
             'replication_status': status.FAILOVER_ERROR},
            {'id': str(uuid.uuid4()), 'status': 'in-use',
             'replication_status': status.ENABLED},
            {'id': str(uuid.uuid4()), 'status': 'available',
             'group_id': in_groups[0]['id'],
             'replication_status': status.FAILOVER_ERROR},
            {'id': str(uuid.uuid4()), 'status': 'available',
             'group_id': in_groups[1]['id'],
             'replication_status': status.ENABLED},
        ]
        in_snapshots = [
            {'id': v['id'], 'volume_id': v['id'], 'status': 'available'}
            for v in in_volumes
        ]
        driver_volumes = {
            v['id'] for v in in_volumes
            if v['replication_status'] not in (status.DISABLED,
                                               status.NOT_CAPABLE)}
        driver_result = [
            {'volume_id': in_volumes[3]['id'],
             'updates': {'status': 'error'}},
            {'volume_id': in_volumes[4]['id'],
             'updates': {'replication_status': status.FAILOVER_ERROR}},
            {'volume_id': in_volumes[5]['id'],
             'updates': {'replication_status': status.FAILED_OVER}},
            {'volume_id': in_volumes[6]['id'],
             'updates': {'replication_status': status.FAILOVER_ERROR}},
            {'volume_id': in_volumes[7]['id'],
             'updates': {'replication_status': status.FAILED_OVER}},
        ]
        out_volumes = [
            {'id': in_volumes[0]['id'], 'status': 'error',
             'replication_status': status.NOT_CAPABLE,
             'previous_status': in_volumes[0]['status']},
            {'id': in_volumes[1]['id'], 'status': 'error',
             'replication_status': status.NOT_CAPABLE,
             'previous_status': in_volumes[1]['status']},
            {'id': in_volumes[2]['id'], 'status': in_volumes[2]['status'],
             'replication_status': status.FAILED_OVER},
            {'id': in_volumes[3]['id'], 'status': 'error',
             'previous_status': in_volumes[3]['status'],
             'replication_status': status.FAILOVER_ERROR},
            {'id': in_volumes[4]['id'], 'status': 'error',
             'previous_status': in_volumes[4]['status'],
             'replication_status': status.FAILOVER_ERROR},
            {'id': in_volumes[5]['id'], 'status': in_volumes[5]['status'],
             'replication_status': status.FAILED_OVER},
            {'id': in_volumes[6]['id'], 'status': 'error',
             'previous_status': in_volumes[6]['status'],
             'replication_status': status.FAILOVER_ERROR},
            {'id': in_volumes[7]['id'], 'status': in_volumes[7]['status'],
             'replication_status': status.FAILED_OVER},
        ]
        out_snapshots = [
            {'id': ov['id'],
             'status': 'error' if ov['status'] == 'error' else 'available'}
            for ov in out_volumes
        ]

        self._test_failover_model_updates(in_volumes, in_snapshots,
                                          driver_volumes, driver_result,
                                          out_volumes, out_snapshots,
                                          in_groups, out_groups,
                                          driver_group_result)

    def test_failback_host_model_updates(self):
        status = fields.ReplicationStatus
        # IDs will be overwritten with UUIDs, but they help follow the code
        in_volumes = [
            {'id': 0, 'status': 'available',
             'replication_status': status.DISABLED},
            {'id': 1, 'status': 'in-use',
             'replication_status': status.NOT_CAPABLE},
            {'id': 2, 'status': 'available',
             'replication_status': status.FAILOVER_ERROR},
            {'id': 3, 'status': 'in-use',
             'replication_status': status.ENABLED},
            {'id': 4, 'status': 'available',
             'replication_status': status.FAILOVER_ERROR},
            {'id': 5, 'status': 'in-use',
             'replication_status': status.FAILED_OVER},
        ]
        # Generate real volume IDs
        for volume in in_volumes:
            volume['id'] = str(uuid.uuid4())
        in_snapshots = [
            {'id': in_volumes[0]['id'], 'volume_id': in_volumes[0]['id'],
             'status': fields.SnapshotStatus.ERROR_DELETING},
            {'id': in_volumes[1]['id'], 'volume_id': in_volumes[1]['id'],
             'status': fields.SnapshotStatus.AVAILABLE},
            {'id': in_volumes[2]['id'], 'volume_id': in_volumes[2]['id'],
             'status': fields.SnapshotStatus.CREATING},
            {'id': in_volumes[3]['id'], 'volume_id': in_volumes[3]['id'],
             'status': fields.SnapshotStatus.DELETING},
            {'id': in_volumes[4]['id'], 'volume_id': in_volumes[4]['id'],
             'status': fields.SnapshotStatus.CREATING},
            {'id': in_volumes[5]['id'], 'volume_id': in_volumes[5]['id'],
             'status': fields.SnapshotStatus.CREATING},
        ]
        driver_volumes = {
            v['id'] for v in in_volumes
            if v['replication_status'] not in (status.DISABLED,
                                               status.NOT_CAPABLE)}
        driver_result = [
            {'volume_id': in_volumes[3]['id'],
             'updates': {'status': 'error'}},
            {'volume_id': in_volumes[4]['id'],
             'updates': {'replication_status': status.FAILOVER_ERROR}},
            {'volume_id': in_volumes[5]['id'],
             'updates': {'replication_status': status.FAILED_OVER}},
        ]
        out_volumes = [
            {'id': in_volumes[0]['id'], 'status': in_volumes[0]['status'],
             'replication_status': in_volumes[0]['replication_status'],
             'previous_status': None},
            {'id': in_volumes[1]['id'], 'status': in_volumes[1]['status'],
             'replication_status': in_volumes[1]['replication_status'],
             'previous_status': None},
            {'id': in_volumes[2]['id'], 'status': in_volumes[2]['status'],
             'replication_status': status.ENABLED},
            {'id': in_volumes[3]['id'], 'status': 'error',
             'previous_status': in_volumes[3]['status'],
             'replication_status': status.FAILOVER_ERROR},
            {'id': in_volumes[4]['id'], 'status': 'error',
             'previous_status': in_volumes[4]['status'],
             'replication_status': status.FAILOVER_ERROR},
            {'id': in_volumes[5]['id'], 'status': in_volumes[5]['status'],
             'replication_status': status.ENABLED},
        ]
        # Snapshot status is preserved except for those that error the failback
        out_snapshots = in_snapshots[:]
        out_snapshots[3]['status'] = fields.SnapshotStatus.ERROR
        out_snapshots[4]['status'] = fields.SnapshotStatus.ERROR

        self._test_failover_model_updates(in_volumes, in_snapshots,
                                          driver_volumes, driver_result,
                                          out_volumes, out_snapshots,
                                          [], [], [],
                                          self.manager.FAILBACK_SENTINEL)

    @mock.patch('cinder.volume.volume_utils.log_unsupported_driver_warning',
                mock.Mock())
    @mock.patch('cinder.volume.volume_utils.require_driver_initialized',
                mock.Mock())
    def test_init_host_with_rpc_clustered_replication(self):
        # These are not OVOs but ORM instances
        cluster = utils.create_cluster(self.context)
        service = utils.create_service(self.context,
                                       {'cluster_name': cluster.name,
                                        'binary': cluster.binary})
        self.assertNotEqual(fields.ReplicationStatus.ENABLED,
                            cluster.replication_status)
        self.assertNotEqual(fields.ReplicationStatus.ENABLED,
                            service.replication_status)

        vol_manager = manager.VolumeManager(
            'cinder.tests.fake_driver.FakeHAReplicatedLoggingVolumeDriver',
            host=service.host, cluster=cluster.name)
        vol_manager.driver = mock.Mock()
        vol_manager.driver.get_volume_stats.return_value = {
            'replication_enabled': True
        }
        vol_manager.init_host_with_rpc()

        cluster_ovo = objects.Cluster.get_by_id(self.context, cluster.id)
        service_ovo = objects.Service.get_by_id(self.context, service.id)

        self.assertEqual(fields.ReplicationStatus.ENABLED,
                         cluster_ovo.replication_status)
        self.assertEqual(fields.ReplicationStatus.ENABLED,
                         service_ovo.replication_status)
