from __future__ import absolute_import, print_function, division
from itertools import product
import time
import unittest

from nose.plugins.skip import SkipTest
import numpy as np
from six.moves import xrange
try:
    import scipy.sparse as sp
    import scipy.sparse
    from scipy.sparse import csr_matrix
except ImportError:
    pass  # The variable enable_sparse will be used to disable the test file.

import theano
from theano import tensor
from theano import sparse
from theano import compile, config, gof
from theano.sparse import enable_sparse
from theano.tensor.basic import _allclose
from theano.tests.unittest_tools import attr

if not enable_sparse:
    raise SkipTest('Optional package SciPy not installed')

from theano.sparse.basic import _is_dense, _is_sparse, _mtypes
from theano.sparse.basic import _is_dense_variable, _is_sparse_variable
from theano.sparse import (
    as_sparse_variable, as_sparse_or_tensor_variable,
    CSR, CSC, CSM, CSMProperties, csm_properties,
    DenseFromSparse,
    SparseType, CSMGrad,
    StructuredDot,
    StructuredDotGradCSC, StructuredDotGradCSR,
    AddSS, AddSD, MulSS, MulSD, Transpose, Neg, Remove0,
    add, mul, structured_dot, transpose,
    csc_from_dense, csr_from_dense, dense_from_sparse,
    Dot, Usmm, sp_ones_like, GetItemScalar, GetItemList, GetItem2Lists,
    SparseFromDense,
    Cast, cast, HStack, VStack, AddSSData, add_s_s_data,
    structured_minimum, structured_maximum, structured_add,
    mul_s_v, structured_add_s_v,
    SamplingDot, sampling_dot,
    Diag, diag, SquareDiagonal, square_diagonal,
    EnsureSortedIndices, ensure_sorted_indices, clean,
    ConstructSparseFromList, construct_sparse_from_list,
    TrueDot, true_dot, eq, neq, le, ge, gt, lt)

# Probability distributions are currently tested in test_sp2.py
# from theano.sparse import (
#    Poisson, poisson, Binomial, Multinomial, multinomial)

from theano.sparse.opt import (StructuredDotCSC, UsmmCscDense, CSMGradC)

from theano.tests import unittest_tools as utt


def as_sparse_format(data, format):
    if format == 'csc':
        return scipy.sparse.csc_matrix(data)
    elif format == 'csr':
        return scipy.sparse.csr_matrix(data)
    else:
        raise NotImplementedError()


def eval_outputs(outputs):
    return compile.function([], outputs)()[0]


# scipy 0.17 will return sparse values in all cases while previous
# version sometimes wouldn't.  This will make everything dense so that
# we can use assert_allclose.
def as_ndarray(val):
    if hasattr(val, 'toarray'):
        return val.toarray()
    return val


def random_lil(shape, dtype, nnz):
    rval = sp.lil_matrix(shape, dtype=dtype)
    huge = 2 ** 30
    for k in range(nnz):
        # set non-zeros in random locations (row x, col y)
        idx = np.random.randint(1, huge+1, size=2) % shape
        value = np.random.rand()
        # if dtype *int*, value will always be zeros!
        if dtype in theano.sparse.integer_dtypes:
            value = int(value * 100)
        # The call to tuple is needed as scipy 0.13.1 do not support
        # ndarray with lenght 2 as idx tuple.
        rval.__setitem__(
            tuple(idx),
            value)
    return rval


def sparse_random_inputs(format, shape, n=1, out_dtype=None, p=0.5, gap=None,
                         explicit_zero=False, unsorted_indices=False):
    """Return a tuple containing everything needed to
    perform a test.

    If `out_dtype` is `None`, theano.config.floatX is
    used.

    :param format: Sparse format.
    :param shape: Shape of data.
    :param n: Number of variable.
    :param out_dtype: dtype of output.
    :param p: Sparsity proportion.
    :param gap: Tuple for the range of the random sample. When
                length is 1, it is assumed to be the exclusive
                max, when `gap` = (`a`, `b`) it provide a sample
                from [a, b[. If `None` is used, it provide [0, 1]
                for float dtypes and [0, 50[ for integer dtypes.
    :param explicit_zero: When True, we add explicit zero in the
                          returned sparse matrix
    :param unsorted_indices: when True, we make sure there is
                             unsorted indices in the returned
                             sparse matrix.
    :return: (variable, data) where both `variable`
             and `data` are list.

    :note: explicit_zero and unsorted_indices was added in Theano 0.6rc4
    """

    if out_dtype is None:
        out_dtype = theano.config.floatX

    assert 0 <= p <= 1
    assert len(shape) == 2
    assert out_dtype in sparse.all_dtypes
    assert gap is None or isinstance(gap, (tuple, list))
    if gap is not None and out_dtype.startswith('u'):
        assert gap[0] >= 0

    def _rand():
        where = np.random.binomial(1, p, size=shape).astype('int8')

        if out_dtype in sparse.discrete_dtypes:
            if not gap:
                value = np.random.randint(50, size=shape)
            elif len(gap) == 2:
                value = np.random.randint(gap[0], gap[1], size=shape)
            else:
                value = np.random.randint(gap[0], size=shape)
        else:
            if not gap:
                value = np.random.random(shape)
            elif len(gap) == 2:
                a, b = gap
                value = a + np.random.random(shape) * (b - a)
            else:
                value = np.random.random(shape) * gap[0]
        return (where * value).astype(out_dtype)

    variable = [getattr(theano.sparse, format + '_matrix')(dtype=out_dtype)
                for k in range(n)]
    data = [getattr(scipy.sparse, format + '_matrix')(_rand(), dtype=out_dtype)
            for k in range(n)]
    if unsorted_indices:
        for idx in range(n):
            d = data[idx]
            d = d[list(range(d.shape[0]))]
            assert not d.has_sorted_indices
            data[idx] = d
    if explicit_zero:
        for idx in range(n):
            assert data[idx].nnz > 1, (
                "can't make a sparse matrix with explicit 0")
            d_idx = np.random.randint(data[idx].nnz)
            data[idx].data[d_idx] = 0

    # numpy 1.5.0 with scipy 0.9.0 have scipy.sparse.XXX_matrix return
    # typenum 10(ulonglong) instead of 8(uint64) event if they are the same!
    # Theano don't like ulonglong type_num
    dtype = np.dtype(out_dtype)  # Convert into dtype object.
    if data[0].dtype.num != dtype.num and dtype.str == data[0].dtype.str:
        data[0].data = theano._asarray(data[0].data, out_dtype)
    assert data[0].dtype.num == dtype.num
    return (variable, data)


def verify_grad_sparse(op, pt, structured=False, *args, **kwargs):
    """
    Wrapper for theano.test.unittest_tools.py:verify_grad wich
    converts sparse variables back and forth.

    Parameters
    ----------
    op
        Op to check.
    pt
        List of inputs to realize the tests.
    structured
        True to tests with a structured grad, False otherwise.
    args
        Other `verify_grad` parameters if any.
    kwargs
        Other `verify_grad` keywords if any.

    Returns
    -------
    None

    """

    def conv_none(x):
        return x

    def conv_csr(ind, indptr, shp):
        def f(spdata):
            return CSR(spdata, ind, indptr, shp)
        return f

    def conv_csc(ind, indptr, shp):
        def f(spdata):
            return CSC(spdata, ind, indptr, shp)
        return f

    iconv = []
    dpt = []

    for p in pt:
        if _is_sparse(p):
            if structured:
                dpt.append(p.data)
            else:
                dpt.append(p.toarray())
            if p.format == 'csr':
                if structured:
                    iconv.append(conv_csr(p.indices[:p.size], p.indptr,
                                          p.shape))
                else:
                    iconv.append(csr_from_dense)
            elif p.format == 'csc':
                if structured:
                    iconv.append(conv_csc(p.indices[:p.size], p.indptr,
                                          p.shape))
                else:
                    iconv.append(csc_from_dense)
            else:
                raise NotImplementedError("No conv for %s" % (p.format,))
        else:
            dpt.append(p)
            iconv.append(conv_none)
    output = op(*[as_sparse_or_tensor_variable(p) for p in pt])
    if isinstance(output, (list, tuple)):
        raise NotImplementedError("verify_grad can't deal with "
                                  "multiple outputs")
    if _is_sparse_variable(output):
        oconv = DenseFromSparse(structured=structured)
    else:
        oconv = conv_none

    def conv_op(*inputs):
        ipt = [conv(i) for i, conv in zip(inputs, iconv)]
        out = op(*ipt)
        return oconv(out)

    return utt.verify_grad(conv_op, dpt, *args, **kwargs)
verify_grad_sparse.E_grad = utt.verify_grad.E_grad


class T_verify_grad_sparse(unittest.TestCase):
    class FailOp(gof.op.Op):
        def __init__(self, structured):
            self.structured = structured

        def __eq__(self, other):
            return (type(self) == type(other)) and \
                self.structured == other.structured

        def __hash__(self):
            return hash(type(self)) ^ hash(self.structured)

        def make_node(self, x):
            x = as_sparse_variable(x)
            return gof.Apply(self, [x], [x.type()])

        def perform(self, node, inputs, outputs):
            (x,) = inputs
            (out,) = outputs
            assert _is_sparse(x)
            out[0] = -x

        def grad(self, inputs, gout):
            (x,) = inputs
            (gz,) = gout
            assert _is_sparse_variable(x) and _is_sparse_variable(gz)
            if self.structured:
                return sp_ones_like(x) * dense_from_sparse(gz),
            else:
                return gz,

        def infer_shape(self, node, shapes):
            return [shapes[0]]

    def test_grad_fail(self):
        self.assertRaises(verify_grad_sparse.E_grad,
                          verify_grad_sparse,
                          self.FailOp(structured=False),
                          [sp.csr_matrix(random_lil((10, 40),
                                                    config.floatX, 3))])

        self.assertRaises(verify_grad_sparse.E_grad,
                          verify_grad_sparse,
                          self.FailOp(structured=True),
                          [sp.csr_matrix(random_lil((10, 40),
                                                    config.floatX, 3))])


class T_transpose(unittest.TestCase):
    def setUp(self):
        utt.seed_rng()

    def test_transpose_csc(self):
        sp = scipy.sparse.csc_matrix(scipy.sparse.eye(5, 3))
        a = as_sparse_variable(sp)
        self.assertFalse(a.data is sp)
        self.assertTrue(a.data.shape == (5, 3))
        self.assertTrue(a.type.dtype == 'float64', a.type.dtype)
        self.assertTrue(a.type.format == 'csc', a.type.format)
        ta = transpose(a)
        self.assertTrue(ta.type.dtype == 'float64', ta.type.dtype)
        self.assertTrue(ta.type.format == 'csr', ta.type.format)

        vta = eval_outputs([ta])
        self.assertTrue(vta.shape == (3, 5))

    def test_transpose_csr(self):
        a = as_sparse_variable(scipy.sparse.csr_matrix(scipy.sparse.eye(5, 3)))
        self.assertTrue(a.data.shape == (5, 3))
        self.assertTrue(a.type.dtype == 'float64')
        self.assertTrue(a.type.format == 'csr')
        ta = transpose(a)
        self.assertTrue(ta.type.dtype == 'float64', ta.type.dtype)
        self.assertTrue(ta.type.format == 'csc', ta.type.format)

        vta = eval_outputs([ta])
        self.assertTrue(vta.shape == (3, 5))


class SparseInferShapeTester(utt.InferShapeTester):
    def test_getitem_2d(self):
        raise SkipTest('infer_shape not implemented for GetItem2d yet')

    def test_getitem_scalar(self):
        x = SparseType('csr', dtype=config.floatX)()
        self._compile_and_check([x],
                                [x[2, 2]],
                                [sp.csr_matrix(random_lil((10, 40),
                                               config.floatX, 3))],
                                GetItemScalar)

    def test_csm(self):
        for sparsetype in ('csr', 'csc'):
            x = tensor.vector()
            y = tensor.ivector()
            z = tensor.ivector()
            s = tensor.ivector()
            call = getattr(sp, sparsetype + '_matrix')
            spm = call(random_lil((300, 400), config.floatX, 5))
            out = CSM(sparsetype)(x, y, z, s)
            self._compile_and_check([x, y, z, s],
                                    [out],
                                    [spm.data, spm.indices, spm.indptr,
                                     spm.shape],
                                    CSM
            )

    def test_csm_grad(self):
        for sparsetype in ('csr', 'csc'):
            x = tensor.vector()
            y = tensor.ivector()
            z = tensor.ivector()
            s = tensor.ivector()
            call = getattr(sp, sparsetype + '_matrix')
            spm = call(random_lil((300, 400), config.floatX, 5))
            out = tensor.grad(dense_from_sparse(
                CSM(sparsetype)(x, y, z, s)
            ).sum(), x)
            self._compile_and_check([x, y, z, s],
                                    [out],
                                    [spm.data, spm.indices, spm.indptr,
                                     spm.shape],
                                    (CSMGrad, CSMGradC)
                                   )

    def test_transpose(self):
        x = SparseType('csr', dtype=config.floatX)()
        self._compile_and_check([x],
                                [x.T],
                                [sp.csr_matrix(random_lil((10, 40),
                                               config.floatX, 3))],
                                Transpose)

    def test_neg(self):
        x = SparseType('csr', dtype=config.floatX)()
        self._compile_and_check([x],
                                [-x],
                                [sp.csr_matrix(random_lil((10, 40),
                                               config.floatX, 3))],
                                Neg)

    def test_add_ss(self):
        x = SparseType('csr', dtype=config.floatX)()
        y = SparseType('csr', dtype=config.floatX)()
        self._compile_and_check([x, y],
                                [x + y],
                                [sp.csr_matrix(random_lil((10, 40),
                                               config.floatX, 3)),
                                 sp.csr_matrix(random_lil((10, 40),
                                               config.floatX, 3))],
                                AddSS)

    def test_add_sd(self):
        x = SparseType('csr', dtype=config.floatX)()
        y = tensor.matrix()
        self._compile_and_check(
                [x, y],
                [x + y],
                [sp.csr_matrix(random_lil((10, 40),
                               config.floatX, 3)),
                 np.random.randn(10, 40).astype(config.floatX)],
                (AddSD, sparse.opt.AddSD_ccode))

    def test_mul_ss(self):
        x = SparseType('csr', dtype=config.floatX)()
        y = SparseType('csr', dtype=config.floatX)()
        self._compile_and_check([x, y],
                                [x * y],
                                [sp.csr_matrix(random_lil((10, 40),
                                               config.floatX, 3)),
                                ] * 2,
                                MulSS)

    def test_mul_sd(self):
        x = SparseType('csr', dtype=config.floatX)()
        y = tensor.matrix()
        self._compile_and_check(
                [x, y],
                [x * y],
                [sp.csr_matrix(random_lil((10, 40),
                               config.floatX, 3)),
                 np.random.randn(10, 40).astype(config.floatX)],
                MulSD, excluding=["local_mul_s_d"])

    def test_remove0(self):
        x = SparseType('csr', dtype=config.floatX)()
        self._compile_and_check([x],
                                [Remove0()(x)],
                                [sp.csr_matrix(random_lil((10, 40),
                                               config.floatX, 3))],
                                Remove0)

    def test_dot(self):
        x = SparseType('csc', dtype=config.floatX)()
        y = SparseType('csc', dtype=config.floatX)()
        self._compile_and_check(
                [x, y],
                [Dot()(x, y)],
                [sp.csc_matrix(random_lil((4, 5),
                               config.floatX, 3)),
                 sp.csc_matrix(random_lil((5, 3),
                               config.floatX, 3))],
                Dot)

    def test_structured_dot(self):
        x = SparseType('csc', dtype=config.floatX)()
        y = SparseType('csc', dtype=config.floatX)()
        self._compile_and_check(
                [x, y],
                [structured_dot(x, y)],
                [sp.csc_matrix(random_lil((4, 5),
                               config.floatX, 3)),
                 sp.csc_matrix(random_lil((5, 3),
                               config.floatX, 3))],
                StructuredDot)

    def test_structured_dot_grad(self):
        # We also need the grad of CSM to be implemetned.
        raise SkipTest('infer_shape not implemented for the grad'
                       ' of structured_dot')
        for format, op in [('csc', StructuredDotGradCSC),
                           ('csr', StructuredDotGradCSR)]:
            x = SparseType(format, dtype=config.floatX)()
            y = SparseType(format, dtype=config.floatX)()
            grads = tensor.grad(dense_from_sparse(structured_dot(x, y)).sum(),
                                [x, y])
            self._compile_and_check(
                    [x, y],
                    [grads[0]],
                    [as_sparse_format(random_lil((4, 5),
                                                 config.floatX, 3), format),
                     as_sparse_format(random_lil((5, 3),
                                                 config.floatX, 3), format)],
                    op)
            self._compile_and_check(
                    [x, y],
                    [grads[1]],
                    [as_sparse_format(random_lil((4, 5),
                                                 config.floatX, 3), format),
                     as_sparse_format(random_lil((5, 3),
                                                 config.floatX, 3), format)],
                    op)

    def test_dense_from_sparse(self):
        x = SparseType('csr', dtype=config.floatX)()
        self._compile_and_check([x],
                                [dense_from_sparse(x)],
                                [sp.csr_matrix(random_lil((10, 40),
                                               config.floatX, 3))],
                                dense_from_sparse.__class__)

    def test_sparse_from_dense(self):
        x = tensor.matrix()
        self._compile_and_check([x],
                                [csc_from_dense(x)],
                                [np.random.randn(10, 40).astype(
                                    config.floatX)],
                                csc_from_dense.__class__)

    def test_sparse_from_list(self):
        x = tensor.matrix('x')
        vals = tensor.matrix('vals')
        ilist = tensor.lvector('ilist')

        out = construct_sparse_from_list(x, vals, ilist)
        self._compile_and_check(
                [x, vals, ilist],
                [out],
                [np.zeros((40, 10), dtype=config.floatX),
                 np.random.randn(12, 10).astype(config.floatX),
                 np.random.randint(low=0, high=40, size=(12,))],
                ConstructSparseFromList
                )


class TestConstructSparseFromList(unittest.TestCase):
    def test_adv_sub1_sparse_grad(self):
        v = theano.tensor.ivector()

        # Assert we don't create a sparse grad by default
        m = theano.tensor.matrix()
        sub = m[v]
        g = theano.grad(sub.sum(), m)
        assert isinstance(g.owner.op, tensor.AdvancedIncSubtensor1)

        # Test that we create a sparse grad when asked
        # USER INTERFACE
        m = theano.tensor.matrix()
        v = theano.tensor.ivector()
        sub = theano.sparse_grad(m[v])
        g = theano.grad(sub.sum(), m)
        assert isinstance(g.owner.op, ConstructSparseFromList)

        # Test that we create a sparse grad when asked
        # Op INTERFACE
        m = theano.tensor.matrix()
        v = theano.tensor.ivector()
        sub = theano.tensor.AdvancedSubtensor1(sparse_grad=True)(m, v)
        g = theano.grad(sub.sum(), m)
        assert isinstance(g.owner.op, ConstructSparseFromList)

        # Test the sparse grad
        valm = np.random.rand(5, 4).astype(config.floatX)
        valv = np.random.randint(0, 5, 10)
        m = theano.tensor.matrix()
        shared_v = theano.shared(valv)

        def fn(m):
            return theano.sparse_grad(m[shared_v])
        verify_grad_sparse(fn, [valm])

    def test_err(self):
        for ndim in [1, 3]:
            t = theano.tensor.TensorType(dtype=config.floatX,
                                         broadcastable=(False,) * ndim)()
            v = theano.tensor.ivector()
            sub = t[v]

            # Assert we don't create a sparse grad by default
            g = theano.grad(sub.sum(), t)
            assert isinstance(g.owner.op, tensor.AdvancedIncSubtensor1)

            # Test that we raise an error, as we can't create a sparse
            # grad from tensors that don't have 2 dimensions.
            sub = theano.sparse_grad(sub)
            self.assertRaises(TypeError, theano.grad, sub.sum(), t)


class T_AddMul(unittest.TestCase):
    def testAddSS(self):
        self._testSS(add)

    def testAddSD(self):
        self._testSD(add)

    def testAddDS(self):
        self._testDS(add)

    def testMulSS(self):
        self._testSS(mul,
                     np.array([[1., 0], [3, 0], [0, 6]]),
                     np.array([[1., 2], [3, 0], [0, 6]]))

    def testMulSD(self):
        self._testSD(mul,
                     np.array([[1., 0], [3, 0], [0, 6]]),
                     np.array([[1., 2], [3, 0], [0, 6]]))

    def testMulDS(self):
        self._testDS(mul,
                     np.array([[1., 0], [3, 0], [0, 6]]),
                     np.array([[1., 2], [3, 0], [0, 6]]))

    def _testSS(self, op, array1=np.array([[1., 0], [3, 0], [0, 6]]),
                array2=np.asarray([[0, 2.], [0, 4], [5, 0]])):
        for mtype1, mtype2 in product(_mtypes, _mtypes):
            for dtype1, dtype2 in [('float64', 'int8'),
                                   ('int8', 'float64'),
                                   ('float64', 'float64'),
                               ]:
                a = mtype1(array1).astype(dtype1)
                aR = as_sparse_variable(a)
                self.assertFalse(aR.data is a)
                self.assertTrue(_is_sparse(a))
                self.assertTrue(_is_sparse_variable(aR))

                b = mtype2(array2).astype(dtype2)
                bR = as_sparse_variable(b)
                self.assertFalse(bR.data is b)
                self.assertTrue(_is_sparse(b))
                self.assertTrue(_is_sparse_variable(bR))

                apb = op(aR, bR)
                self.assertTrue(_is_sparse_variable(apb))

                self.assertTrue(apb.type.format == aR.type.format, apb.type.format)

                val = eval_outputs([apb])
                self.assertTrue(val.shape == (3, 2))
                if op is add:
                    self.assertTrue(np.all(val.todense() == (array1 + array2)))
                    if dtype1.startswith('float') and dtype2.startswith('float'):
                        verify_grad_sparse(op, [a, b], structured=False)
                elif op is mul:
                    self.assertTrue(np.all(val.todense()
                                              == (array1 * array2)))
                    if dtype1.startswith('float') and dtype2.startswith('float'):
                        verify_grad_sparse(op, [a, b], structured=False)

    def _testSD(self, op, array1=np.array([[1., 0], [3, 0], [0, 6]]),
                array2=np.asarray([[0, 2.], [0, 4], [5, 0]])):
        for mtype in _mtypes:
            for a in [np.array(array1), tensor.as_tensor_variable(array1),
                      theano.shared(array1)]:
                for dtype1, dtype2 in [('float64', 'int8'),
                                       ('int8', 'float64'),
                                       # Needed to test the grad
                                       ('float32', 'float64'),
                                   ]:
                    a = a.astype(dtype1)
                    b = mtype(array2).astype(dtype2)
                    bR = as_sparse_variable(b)
                    self.assertFalse(bR.data is b)  # constants are copied
                    self.assertTrue(_is_sparse(b))
                    self.assertTrue(_is_sparse_variable(bR))

                    apb = op(a, bR)

                    val = eval_outputs([apb])
                    self.assertTrue(val.shape == (3, 2))
                    if op is add:
                        self.assertTrue(_is_dense_variable(apb))
                        self.assertTrue(np.all(val == (array1 + b)))
                        ans = np.array([[1., 2], [3, 4], [5, 6]])
                        self.assertTrue(np.all(val == ans))
                        if isinstance(a, theano.Constant):
                            a = a.data
                        if getattr(a, 'owner', None):
                            continue
                        if dtype1.startswith('float') and dtype2.startswith('float'):
                            verify_grad_sparse(op, [a, b], structured=True)
                    elif op is mul:
                        self.assertTrue(_is_sparse_variable(apb))
                        self.assertTrue(np.all(val.todense() == (b.multiply(array1))))
                        self.assertTrue(np.all(val.todense() == np.array(
                            [[1, 0], [9, 0], [0, 36]])))
                        if isinstance(a, theano.Constant):
                            a = a.data
                        if getattr(a, 'owner', None):
                            continue
                        if dtype1.startswith('float') and dtype2.startswith('float'):
                            verify_grad_sparse(op, [a, b], structured=False)

    def _testDS(self, op, array1=np.array([[1., 0], [3, 0], [0, 6]]),
                array2=np.asarray([[0, 2.], [0, 4], [5, 0]])):
        for mtype in _mtypes:
            for b in [np.asarray(array2),
                      tensor.as_tensor_variable(array2),
                      theano.shared(array2)]:
                for dtype1, dtype2 in [('float64', 'int8'),
                                       ('int8', 'float64'),
                                   ]:
                    a = mtype(array1).astype(dtype1)
                    aR = as_sparse_variable(a)
                    self.assertFalse(aR.data is a)
                    self.assertTrue(_is_sparse(a))
                    self.assertTrue(_is_sparse_variable(aR))
                    b = b.astype(dtype2)

                    apb = op(aR, b)

                    val = eval_outputs([apb])
                    self.assertTrue(val.shape == (3, 2))
                    if op is add:
                        self.assertTrue(_is_dense_variable(apb))
                        self.assertTrue(np.all(val == (a + array2)))
                        ans = np.array([[1., 2], [3, 4], [5, 6]])
                        self.assertTrue(np.all(val == ans))
                        if isinstance(b, theano.Constant):
                            b = b.data
                        if dtype1.startswith('float') and dtype2.startswith('float'):
                            verify_grad_sparse(op, [a, b], structured=True)
                    elif op is mul:
                        self.assertTrue(_is_sparse_variable(apb))
                        ans = np.array([[1, 0], [9, 0], [0, 36]])
                        self.assertTrue(np.all(val.todense() == (a.multiply(array2))))
                        self.assertTrue(np.all(val.todense() == ans))
                        if isinstance(b, theano.Constant):
                            b = b.data
                        if dtype1.startswith('float') and dtype2.startswith('float'):
                            verify_grad_sparse(op, [a, b], structured=False)


class test_comparison(unittest.TestCase):
    def setUp(self):
        utt.seed_rng()

    # took from tensor basic_test.py
    def _rand_ranged(self, min, max, shape):
        return np.asarray(np.random.rand(*shape) * (max - min) + min,
                         dtype=config.floatX)

    tests = [lambda x, y: x > y, lambda x, y: x < y,
                lambda x, y: x >= y, lambda x, y: x <= y]

    testsDic = {gt: lambda x, y: x > y, lt: lambda x, y: x < y,
                ge: lambda x, y: x >= y, le: lambda x, y: x <= y}

    def __generalized_ss_test(self, theanop, symbolicType, testOp, scipyType):

        scipy_ver = [int(n) for n in scipy.__version__.split('.')[:2]]

        if (bool(scipy_ver < [0, 13])):
            raise SkipTest("comparison operators need newer release of scipy")

        x = symbolicType()
        y = symbolicType()

        op = theanop(x, y)

        f = theano.function([x, y], op)

        m1 = scipyType(random_lil((10, 40), config.floatX, 3))
        m2 = scipyType(random_lil((10, 40), config.floatX, 3))

        self.assertTrue(np.array_equal(f(m1, m2).data, testOp(m1, m2).data))

    def __generalized_sd_test(self, theanop, symbolicType, testOp, scipyType):

        scipy_ver = [int(n) for n in scipy.__version__.split('.')[:2]]

        if (bool(scipy_ver < [0, 13])):
            raise SkipTest("comparison operators need newer release of scipy")

        x = symbolicType()
        y = theano.tensor.matrix()

        op = theanop(x, y)

        f = theano.function([x, y], op)

        m1 = scipyType(random_lil((10, 40), config.floatX, 3))
        m2 = self._rand_ranged(1000, -1000, [10, 40])

        self.assertTrue(np.array_equal(f(m1, m2).data, testOp(m1, m2).data))

    def __generalized_ds_test(self, theanop, symbolicType, testOp, scipyType):

        scipy_ver = [int(n) for n in scipy.__version__.split('.')[:2]]

        if (bool(scipy_ver < [0, 13])):
            raise SkipTest("comparison operators need newer release of scipy")

        x = symbolicType()
        y = theano.tensor.matrix()

        op = theanop(y, x)

        f = theano.function([y, x], op)

        m1 = scipyType(random_lil((10, 40), config.floatX, 3))
        m2 = self._rand_ranged(1000, -1000, [10, 40])

        self.assertTrue(np.array_equal(f(m2, m1).data, testOp(m2, m1).data))

    def test_ss_csr_comparison(self):

        for op in self.tests:
            self.__generalized_ss_test(op, sparse.csr_matrix,
                              op, sp.csr_matrix)

    def test_ss_csc_comparison(self):

        for op in self.tests:
            self.__generalized_ss_test(op, sparse.csc_matrix,
                              op, sp.csc_matrix)

    def test_sd_csr_comparison(self):

        for op in self.tests:
            self.__generalized_sd_test(op, sparse.csr_matrix,
                              op, sp.csr_matrix)

    def test_sd_csc_comparison(self):

        for op in self.tests:
            self.__generalized_sd_test(op, sparse.csc_matrix,
                              op, sp.csc_matrix)

    def test_ds_csc_comparison(self):

        for op in self.testsDic:
            self.__generalized_ds_test(op, sparse.csc_matrix,
                              self.testsDic[op], sp.csc_matrix)

    def test_ds_csr_comparison(self):

        for op in self.testsDic:
            self.__generalized_ds_test(op, sparse.csr_matrix,
                              self.testsDic[op], sp.csr_matrix)

    def test_equality_case(self):
        """
        Test assuring normal behaviour when values
        in the matrices are equal
        """

        scipy_ver = [int(n) for n in scipy.__version__.split('.')[:2]]

        if (bool(scipy_ver < [0, 13])):
            raise SkipTest("comparison operators need newer release of scipy")

        x = sparse.csc_matrix()
        y = theano.tensor.matrix()

        m1 = sp.csc_matrix((2, 2), dtype=theano.config.floatX)
        m2 = np.asarray([[0, 0], [0, 0]], dtype=theano.config.floatX)

        for func in self.testsDic:

            op = func(y, x)
            f = theano.function([y, x], op)

            self.assertTrue(np.array_equal(f(m2, m1),
                                              self.testsDic[func](m2, m1)))


class T_conversion(unittest.TestCase):
    def setUp(self):
        utt.seed_rng()

    if 0:
        def test0(self):
            a = tensor.as_tensor_variable(np.random.rand(5))
            s = csc_from_dense(a)
            val = eval_outputs([s])
            self.assertTrue(str(val.dtype) == 'float64')
            self.assertTrue(val.format == 'csc')

    if 0:
        def test1(self):
            a = tensor.as_tensor_variable(np.random.rand(5))
            s = csr_from_dense(a)
            val = eval_outputs([s])
            self.assertTrue(str(val.dtype) == 'float64')
            self.assertTrue(val.format == 'csr')

    def test_dense_from_sparse(self):
        # call dense_from_sparse
        for t in _mtypes:
            s = t(scipy.sparse.identity(5))
            s = as_sparse_variable(s)
            d = dense_from_sparse(s)
            val = eval_outputs([d])
            self.assertTrue(str(val.dtype) == s.dtype)
            self.assertTrue(np.all(val[0] == [1, 0, 0, 0, 0]))

    def test_todense(self):
        # call sparse_var.todense()
        for t in _mtypes:
            s = t(scipy.sparse.identity(5))
            s = as_sparse_variable(s)
            d = s.toarray()
            val = eval_outputs([d])
            self.assertTrue(str(val.dtype) == s.dtype)
            self.assertTrue(np.all(val[0] == [1, 0, 0, 0, 0]))

    @staticmethod
    def check_format_ndim(format, ndim):
        x = tensor.tensor(
                dtype=config.floatX,
                broadcastable=([False] * ndim),
                name='x')

        s = SparseFromDense(format)(x)
        s_m = - s
        d = dense_from_sparse(s_m)
        c = d.sum()
        g = tensor.grad(c, x)
        f = theano.function([x], [s, g])
        f(np.array(0, dtype=config.floatX, ndmin=ndim))
        f(np.array(7, dtype=config.floatX, ndmin=ndim))

    def test_format_ndim(self):
        for format in 'csc', 'csr':
            for ndim in 0, 1, 2:
                self.check_format_ndim(format, ndim)

            self.assertRaises(TypeError, self.check_format_ndim, format, 3)
            self.assertRaises(TypeError, self.check_format_ndim, format, 4)


class test_csm_properties(unittest.TestCase):
    def setUp(self):
        utt.seed_rng()

    def test_csm_properties_grad(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csc', 'csr']:
            for dtype in ['float32', 'float64']:
                spmat = sp_types[format](random_lil((4, 3), dtype, 3))

                verify_grad_sparse(lambda *x: CSMProperties()(*x)[0], [spmat],
                                   structured=True)

                verify_grad_sparse(lambda *x: CSMProperties()(*x)[1], [spmat],
                                   structured=True)

                verify_grad_sparse(lambda *x: CSMProperties()(*x)[2], [spmat],
                                   structured=True)

                verify_grad_sparse(lambda *x: CSMProperties()(*x)[2], [spmat],
                                   structured=True)

    def test_csm_properties(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csc', 'csr']:
            for dtype in ['float32', 'float64']:
                x = SparseType(format, dtype=dtype)()
                f = theano.function([x], csm_properties(x))

                spmat = sp_types[format](random_lil((4, 3), dtype, 3))

                data, indices, indptr, shape = f(spmat)

                assert np.all(data == spmat.data)
                assert np.all(indices == spmat.indices)
                assert np.all(indptr == spmat.indptr)
                assert np.all(shape == spmat.shape)


class test_csm(unittest.TestCase):
    def setUp(self):
        utt.seed_rng()

    def test_csm_grad(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csc', 'csr']:
            for dtype in ['float32', 'float64']:
                spmat = sp_types[format](random_lil((4, 3), dtype, 3))

                verify_grad_sparse(lambda x: CSM(format)(x, spmat.indices,
                    spmat.indptr, np.asarray(spmat.shape, 'int32')),
                    [spmat.data], structured=True)

    def test_csm_sparser(self):
        """
        Test support for gradients sparser than the input.
        """
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csc', 'csr']:
            for dtype in ['float32', 'float64']:
                x = tensor.tensor(dtype=dtype, broadcastable=(False,))
                y = tensor.ivector()
                z = tensor.ivector()
                s = tensor.ivector()

                a = as_sparse_variable(sp_types[format](random_lil((4, 3),
                                                                   dtype, 1)))

                f = theano.function([x, y, z, s],
                                    tensor.grad(dense_from_sparse(
                                        a * CSM(format)(x, y, z, s)).sum(), x))

                spmat = sp_types[format](random_lil((4, 3), dtype, 3))

                res = f(spmat.data, spmat.indices, spmat.indptr,
                        np.asarray(spmat.shape, 'int32'))

                assert len(spmat.data) == len(res)

    def test_csm_unsorted(self):
        """
        Test support for gradients of unsorted inputs.
        """
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csr', 'csc', ]:
            for dtype in ['float32', 'float64']:
                x = tensor.tensor(dtype=dtype, broadcastable=(False,))
                y = tensor.ivector()
                z = tensor.ivector()
                s = tensor.ivector()
                # Sparse advanced indexing produces unsorted sparse matrices
                a = sparse_random_inputs(format, (4, 3), out_dtype=dtype,
                                         unsorted_indices=True)[1][0]
                # Make sure it's unsorted
                assert not a.has_sorted_indices
                def my_op(x):
                    y = tensor.constant(a.indices)
                    z = tensor.constant(a.indptr)
                    s = tensor.constant(a.shape)
                    return tensor.sum(
                        dense_from_sparse(CSM(format)(x, y, z, s) * a))
                verify_grad_sparse(my_op, [a.data])

    def test_csm(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csc', 'csr']:
            for dtype in ['float32', 'float64']:
                x = tensor.tensor(dtype=dtype, broadcastable=(False,))
                y = tensor.ivector()
                z = tensor.ivector()
                s = tensor.ivector()
                f = theano.function([x, y, z, s], CSM(format)(x, y, z, s))

                spmat = sp_types[format](random_lil((4, 3), dtype, 3))

                res = f(spmat.data, spmat.indices, spmat.indptr,
                        np.asarray(spmat.shape, 'int32'))

                assert np.all(res.data == spmat.data)
                assert np.all(res.indices == spmat.indices)
                assert np.all(res.indptr == spmat.indptr)
                assert np.all(res.shape == spmat.shape)


class test_structureddot(unittest.TestCase):
    def setUp(self):
        utt.seed_rng()

    def test_structureddot_csc_grad(self):

        # shortcut: testing csc in float32, testing csr in float64

        # allocate a random sparse matrix
        spmat = sp.csc_matrix(random_lil((4, 3), 'float32', 3))

        mat = np.asarray(np.random.randn(3, 2), 'float32')

        verify_grad_sparse(structured_dot, [spmat, mat], structured=True)

        def buildgraph_T(spmat, mat):
            return structured_dot(mat.T, spmat.T)

        verify_grad_sparse(buildgraph_T, [spmat, mat], structured=True)

    def test_structureddot_csr_grad(self):

        # shortcut: testing csc in float32, testing csr in float64

        # allocate a random sparse matrix
        spmat = sp.csr_matrix(random_lil((4, 3), 'float64', 3))

        mat = np.asarray(np.random.randn(3, 2), 'float64')

        verify_grad_sparse(structured_dot, [spmat, mat], structured=True)

        def buildgraph_T(spmat, mat):
            return structured_dot(mat.T, spmat.T)

        verify_grad_sparse(buildgraph_T, [spmat, mat], structured=True)

    def test_upcast(self):

        typenames = ('float32', 'int64', 'int8', 'int32',
                     'int16', 'float64', 'complex64', 'complex128')
        for dense_dtype in typenames:
            for sparse_dtype in typenames:
                correct_dtype = theano.scalar.upcast(sparse_dtype, dense_dtype)
                a = SparseType('csc', dtype=sparse_dtype)()
                b = tensor.matrix(dtype=dense_dtype)
                d = structured_dot(a, b)
                assert d.type.dtype == correct_dtype

                # compile and run a function

                f = theano.function([a, b], d)

                M, N, K, nnz = (4, 3, 5, 3)
                spmat = sp.csc_matrix(random_lil((M, N), sparse_dtype, nnz))
                # the following madness is necessary to workaround
                # an intc vs. int32 bug.
                # The lil makes an intc on my computer when sparse_dtype
                # is int32.
                spmat.dtype = np.dtype(sparse_dtype)
                mat = np.asarray(np.random.randn(N, K) * 9,
                                    dtype=dense_dtype)
                # print 'DTYPES', sparse_dtype, dense_dtype
                # print 'sym types', a.type, b.type
                # print 'dtype strings', spmat.dtype, mat.dtype
                # print 'numpy dtype num', mat.dtype.num
                # print 'scipy dtype num', spmat.data.dtype.num
                theano_result = f(spmat, mat)
                scipy_result = spmat * mat
                assert theano_result.shape == scipy_result.shape
                assert theano_result.dtype == scipy_result.dtype
                utt.assert_allclose(scipy_result, theano_result)

    def test_opt_unpack(self):
        #
        # Test that a graph involving
        # structured_dot(assembled_csc_matrix) is optimized to be just
        # a structured_dot_csc Op and no assembly of a csc_matrix.
        #
        # The optimization from structured_dot -> structured_dot_csc
        # is currently disabled, So this test is not expected to pass

        return
        #
        kerns = tensor.Tensor(dtype='int64', broadcastable=[False])('kerns')
        spmat = sp.lil_matrix((4, 6), dtype='int64')
        for i in range(5):
            # set non-zeros in random locations (row x, col y)
            x = np.floor(np.random.rand() * spmat.shape[0])
            y = np.floor(np.random.rand() * spmat.shape[1])
            spmat[x, y] = np.random.rand() * 10
        spmat = sp.csc_matrix(spmat)

        images = tensor.Tensor(dtype='float32',
                               broadcastable=[False, False])('images')

        cscmat = CSC(kerns, spmat.indices[:spmat.size],
                     spmat.indptr, spmat.shape)
        f = theano.function([kerns, images], structured_dot(cscmat, images.T))

        sdcscpresent = False
        for node in f.maker.fgraph.toposort():
            # print node.op
            assert not isinstance(node.op, CSM)
            assert not isinstance(node.op, CSMProperties)
            if isinstance(f.maker.fgraph.toposort()[1].op, StructuredDotCSC):
                sdcscpresent = True
        assert sdcscpresent

        kernvals = np.array(spmat.data[:spmat.size])
        # print 'kdtype', kernvals.dtype, kernvals.shape,
        # print kernvals.ndim, kernvals.dtype.num
        # print 'type of kernvals = ', kernvals.dtype
        bsize = 3
        imvals = 1.0 * np.array(np.arange(bsize * spmat.shape[1]).\
                                   reshape(bsize, spmat.shape[1]),
                                   dtype='float32')
        outvals = f(kernvals, imvals)
        # print outvals

    def test_dot_sparse_sparse(self):
        # test dot for 2 input sparse matrix
        sparse_dtype = 'float64'
        sp_mat = {'csc': sp.csc_matrix,
                  'csr': sp.csr_matrix,
                  'bsr': sp.csr_matrix}

        for sparse_format_a in ['csc', 'csr', 'bsr']:
            for sparse_format_b in ['csc', 'csr', 'bsr']:
                a = SparseType(sparse_format_a, dtype=sparse_dtype)()
                b = SparseType(sparse_format_b, dtype=sparse_dtype)()
                d = theano.dot(a, b)
                f = theano.function([a, b], theano.Out(d, borrow=True))
                topo = f.maker.fgraph.toposort()
                for M, N, K, nnz in [(4, 3, 2, 3),
                                     (40, 30, 20, 3),
                                     (40, 30, 20, 30),
                                     (400, 3000, 200, 6000),
                                 ]:
                    a_val = sp_mat[sparse_format_a](
                        random_lil((M, N), sparse_dtype, nnz))
                    b_val = sp_mat[sparse_format_b](
                        random_lil((N, K), sparse_dtype, nnz))
                    f(a_val, b_val)

    def test_csc_correct_output_faster_than_scipy(self):
        sparse_dtype = 'float64'
        dense_dtype = 'float64'

        a = SparseType('csc', dtype=sparse_dtype)()
        b = tensor.matrix(dtype=dense_dtype)
        d = theano.dot(a, b)
        f = theano.function([a, b], theano.Out(d, borrow=True))

        for M, N, K, nnz in [(4, 3, 2, 3),
                             (40, 30, 20, 3),
                             (40, 30, 20, 30),
                             (400, 3000, 200, 6000),
                         ]:
            spmat = sp.csc_matrix(random_lil((M, N), sparse_dtype, nnz))
            mat = np.asarray(np.random.randn(N, K), dense_dtype)
            theano_times = []
            scipy_times = []
            for i in xrange(5):
                t0 = time.time()
                theano_result = f(spmat, mat)
                t1 = time.time()
                scipy_result = spmat * mat
                t2 = time.time()

                theano_times.append(t1 - t0)
                scipy_times.append(t2 - t1)

            theano_time = np.min(theano_times)
            scipy_time = np.min(scipy_times)

            speedup = scipy_time / theano_time
            # print scipy_times
            # print theano_times
            # print ('M=%(M)s N=%(N)s K=%(K)s nnz=%(nnz)s theano_time'
            #       '=%(theano_time)s speedup=%(speedup)s') % locals()

            # fail if Theano is slower than scipy by more than a certain amount
            overhead_tol = 0.003  # seconds overall
            overhead_rtol = 1.2  # times as long
            utt.assert_allclose(scipy_result, theano_result)
            if not theano.config.mode in ["DebugMode", "DEBUG_MODE"]:
                self.assertFalse(theano_time > overhead_rtol * scipy_time +
                                 overhead_tol)

    def test_csr_correct_output_faster_than_scipy(self):

        # contrast with test_grad, we put csr in float32, csc in float64

        sparse_dtype = 'float32'
        dense_dtype = 'float32'

        a = SparseType('csr', dtype=sparse_dtype)()
        b = tensor.matrix(dtype=dense_dtype)
        d = theano.dot(a, b)
        f = theano.function([a, b], d)

        for M, N, K, nnz in [(4, 3, 2, 3),
                             (40, 30, 20, 3),
                             (40, 30, 20, 30),
                             (400, 3000, 200, 6000),
                         ]:
            spmat = sp.csr_matrix(random_lil((M, N), sparse_dtype, nnz))
            mat = np.asarray(np.random.randn(N, K), dense_dtype)
            t0 = time.time()
            theano_result = f(spmat, mat)
            t1 = time.time()
            scipy_result = spmat * mat
            t2 = time.time()

            theano_time = t1 - t0
            scipy_time = t2 - t1
            # print 'theano took', theano_time,
            # print 'scipy took', scipy_time
            overhead_tol = 0.002  # seconds
            overhead_rtol = 1.1  # times as long
            utt.assert_allclose(scipy_result, theano_result)
            if (not theano.config.mode in ["DebugMode", "DEBUG_MODE"] and
                theano.config.cxx):
                    self.assertFalse(
                        theano_time > overhead_rtol * scipy_time + overhead_tol,
                        (theano_time,
                         overhead_rtol * scipy_time + overhead_tol,
                         scipy_time, overhead_rtol, overhead_tol))


class DotTests(utt.InferShapeTester):
    def setUp(self):
        super(DotTests, self).setUp()
        x_size = (10, 100)
        y_size = (100, 1000)
        utt.seed_rng()

        self.x_csr = scipy.sparse.csr_matrix(
            np.random.binomial(1, 0.5, x_size), dtype=theano.config.floatX)
        self.x_csc = scipy.sparse.csc_matrix(
            np.random.binomial(1, 0.5, x_size), dtype=theano.config.floatX)
        self.y = np.asarray(np.random.uniform(-1, 1, y_size),
                               dtype=theano.config.floatX)
        self.y_csr = scipy.sparse.csr_matrix(
            np.random.binomial(1, 0.5, y_size), dtype=theano.config.floatX)
        self.y_csc = scipy.sparse.csc_matrix(
            np.random.binomial(1, 0.5, y_size), dtype=theano.config.floatX)
        self.v_10 = np.asarray(np.random.uniform(-1, 1, 10),
                                  dtype=theano.config.floatX)
        self.v_100 = np.asarray(np.random.uniform(-1, 1, 100),
                                   dtype=theano.config.floatX)

    def test_csr_dense(self):
        x = theano.sparse.csr_matrix('x')
        y = theano.tensor.matrix('y')
        v = theano.tensor.vector('v')

        for (x, y, x_v, y_v) in [(x, y, self.x_csr, self.y),
                                 (x, v, self.x_csr, self.v_100),
                                 (v, x, self.v_10, self.x_csr)]:
            f_a = theano.function([x, y], theano.sparse.dot(x, y))
            f_b = lambda x, y: x * y

            utt.assert_allclose(f_a(x_v, y_v), f_b(x_v, y_v))

            # Test infer_shape
            self._compile_and_check([x, y], [theano.sparse.dot(x, y)],
                                    [x_v, y_v],
                                    (Dot, Usmm, UsmmCscDense))

    def test_csc_dense(self):
        x = theano.sparse.csc_matrix('x')
        y = theano.tensor.matrix('y')
        v = theano.tensor.vector('v')

        for (x, y, x_v, y_v) in [(x, y, self.x_csc, self.y),
                                 (x, v, self.x_csc, self.v_100),
                                 (v, x, self.v_10, self.x_csc)]:

            f_a = theano.function([x, y], theano.sparse.dot(x, y))
            f_b = lambda x, y: x * y

            utt.assert_allclose(f_a(x_v, y_v), f_b(x_v, y_v))

            # Test infer_shape
            self._compile_and_check([x, y], [theano.sparse.dot(x, y)],
                                    [x_v, y_v],
                                    (Dot, Usmm, UsmmCscDense))

    def test_sparse_sparse(self):
        for d1, d2 in [('float32', 'float32'),
                       ('float32', 'float64'),
                       ('float64', 'float32'),
                       ('float64', 'float64'),
                       ('float32', 'int16'),
                       ('float32', 'complex64'),
                       ]:
            for x_f, y_f in [('csc', 'csc'),
                             ('csc', 'csr'),
                             ('csr', 'csc'),
                             ('csr', 'csr'),
                             ]:
                x = theano.sparse.SparseType(format=x_f, dtype=d1)('x')
                y = theano.sparse.SparseType(format=x_f, dtype=d2)('x')

                f_a = lambda x, y: x * y
                f_b = theano.function([x, y], theano.sparse.dot(x, y))

                vx = getattr(self, 'x_' + x_f).astype(d1)
                vy = getattr(self, 'y_' + y_f).astype(d2)
                utt.assert_allclose(f_a(vx, vy).toarray(), f_b(vx, vy))

                # Test infer_shape
                f_a = theano.function([x, y], theano.sparse.dot(x, y).shape)
                f_b = lambda x, y: (x * y).shape
                assert np.all(f_a(vx, vy) == f_b(vx, vy))
                topo = f_a.maker.fgraph.toposort()
                if theano.config.mode != 'FAST_COMPILE':
                    nb = 0
                else:
                    nb = 1
                assert sum([isinstance(node.op, (Dot, Usmm, UsmmCscDense))
                            for node in topo]) == nb

    def test_cuda(self):
        import theano.sandbox.cuda as cuda
        if not cuda.cuda_available:
            raise SkipTest("Optional package cuda not available")

        a = sparse.csr_matrix('a', dtype='float32')
        b = cuda.float32_shared_constructor(
            np.random.rand(3, 4).astype('float32'))
        d = sparse.dot(a, b)
        f = theano.function([a], d)

        a_val = scipy.sparse.csr_matrix(random_lil((5, 3), 'float32', 5))
        d_theano = f(a_val)
        d_numpy = a_val * b.get_value()
        utt.assert_allclose(d_numpy, d_theano)

    def test_int32_dtype(self):
        # Reported on the theano-user mailing-list:
        # https://groups.google.com/d/msg/theano-users/MT9ui8LtTsY/rwatwEF9zWAJ
        size = 9
        intX = 'int32'

        C = tensor.matrix('C', dtype=intX)
        I = tensor.matrix('I', dtype=intX)

        fI = I.flatten()
        data = tensor.ones_like(fI)
        indptr = tensor.arange(data.shape[0] + 1, dtype='int32')

        m1 = sparse.CSR(data, fI, indptr, (8, size))
        m2 = sparse.dot(m1, C)
        y = m2.reshape(shape=(2, 4, 9), ndim=3)

        f = theano.function(inputs=[I, C], outputs=y)
        i = np.asarray([[4, 3, 7, 7], [2, 8, 4, 5]], dtype=intX)
        a = np.asarray(np.random.randint(0, 100, (size, size)),
                          dtype=intX)
        f(i, a)

    def test_csr_dense_grad(self):

        # shortcut: testing csc in float32, testing csr in float64

        # allocate a random sparse matrix
        spmat = sp.csr_matrix(random_lil((4, 3), 'float64', 3))

        mat = np.asarray(np.random.randn(2, 4), 'float64')

        def buildgraph_T(mat):
            return Dot()(mat, spmat)

        theano.tests.unittest_tools.verify_grad(buildgraph_T, [mat])


class UsmmTests(unittest.TestCase):
    """ Test the Usmm and UsmmCscDense class and related optimization """
    def setUp(self):
        x_size = (10, 100)
        y_size = (100, 200)
        z_size = (x_size[0], y_size[1])

        self.rng = np.random.RandomState(seed=utt.fetch_seed())
        self.x = np.asarray(self.rng.binomial(1, 0.5, x_size),
                               dtype=theano.config.floatX)
        self.y = np.asarray(self.rng.uniform(-1, 1, y_size),
                               dtype=theano.config.floatX)
        self.z = np.asarray(self.rng.uniform(-1, 1, z_size),
                               dtype=theano.config.floatX)

    # this is slow, but it's the only test for the op.
    def test(self):
        def mat(format, name, dtype):
            if format == 'dense':
                return theano.tensor.matrix(name, dtype=dtype)
            else:
                return theano.sparse.matrix(format, name, dtype=dtype)

        params = product(*([['float32', 'float64', 'int16', 'complex64']] * 4 +
                           [['dense', 'csc', 'csr']] * 2))

        # All test are too slow, so we randomly take 100 of them.
        # The buildbot change the seed, so we will finish by running them all.
        # As of this writing they where all passing.
        #params = self.rng.permutation(list(params))[:500]

        for dtype1, dtype2, dtype3, dtype4, format1, format2 in params:
            if format1 == 'dense' and format2 == 'dense':
                # Usmm won't be used!
                continue
            x = mat(format1, 'x', dtype1)
            y = mat(format2, 'y', dtype2)
            a = theano.tensor.scalar('a', dtype=dtype3)
            z = theano.shared(np.asarray(self.z, dtype=dtype4).copy())

            f_b = lambda z, a, x, y: z - a * (x * y)
            x_data = np.asarray(self.x, dtype=dtype1)
            if format1 != 'dense':
                x_data = as_sparse_format(x_data, format1)
            y_data = np.asarray(self.y, dtype=dtype2)
            if format2 != 'dense':
                y_data = as_sparse_format(y_data, format2)
            a_data = np.asarray(1.5, dtype=dtype3)
            z_data = np.asarray(self.z, dtype=dtype4)

            f_b_out = f_b(z_data, a_data, x_data, y_data)

            # Can it work inplace?
            inplace = dtype4 == theano.scalar.upcast(dtype1, dtype2, dtype3)

            # To make it easier to check the toposort
            mode = theano.compile.mode.get_default_mode().excluding('fusion')

            if inplace:
                updates = [(z, z - a * theano.sparse.dot(x, y))]
                f_a = theano.function([a, x, y], [],
                                      updates=updates,
                                      mode=mode)
                f_a(a_data, x_data, y_data)
                f_a_out = z.get_value(borrow=True)
            else:
                f_a = theano.function([a, x, y],
                                      z - a * theano.sparse.dot(x, y),
                                      mode=mode)
                # In DebugMode there is a strange difference with complex
                # So we raise a little the threshold a little.
                try:
                    orig_atol = theano.tensor.basic.float64_atol
                    orig_rtol = theano.tensor.basic.float64_rtol
                    theano.tensor.basic.float64_atol = 1e-7
                    theano.tensor.basic.float64_rtol = 1e-6
                    f_a_out = f_a(a_data, x_data, y_data)
                finally:
                    theano.tensor.basic.float64_atol = orig_atol
                    theano.tensor.basic.float64_rtol = orig_rtol

            # As we do a dot product of 2 vector of 100 element,
            # This mean we can have 2*100*eps abs error.
            if f_a_out.dtype in ['float64', 'complex128']:
                atol = 3e-8
                rtol = 1e-5
            else:
                atol = None
                rtol = None
            utt.assert_allclose(f_a_out, f_b_out, rtol=rtol, atol=atol)
            topo = f_a.maker.fgraph.toposort()
            up = theano.scalar.upcast(dtype1, dtype2, dtype3, dtype4)

            fast_compile = theano.config.mode == "FAST_COMPILE"

            if not theano.config.blas.ldflags:
                # Usmm should not be inserted, because it relies on BLAS
                assert len(topo) == 4, topo
                assert isinstance(topo[0].op, theano.sparse.Dot)
                assert isinstance(topo[1].op, theano.tensor.DimShuffle)
                assert (isinstance(topo[2].op, theano.tensor.Elemwise) and
                        isinstance(topo[2].op.scalar_op, theano.scalar.Mul))
                assert (isinstance(topo[3].op, theano.tensor.Elemwise) and
                        isinstance(topo[3].op.scalar_op, theano.scalar.Sub))
            elif (y.type.dtype == up and format1 == 'csc'
                    and format2 == 'dense' and not fast_compile
                    and theano.config.cxx and up in ('float32', 'float64')):
                # The op UsmmCscDense should be inserted
                assert (sum([isinstance(node.op, tensor.Elemwise) and
                             isinstance(node.op.scalar_op,
                                        theano.scalar.basic.Cast)
                             for node in topo]) == len(topo) - 5)
                new_topo = []
                for node in topo:
                    if not (isinstance(node.op, tensor.Elemwise) and
                       isinstance(node.op.scalar_op,
                                  theano.scalar.basic.Cast)):
                        new_topo.append(node)
                topo = new_topo
                assert len(topo) == 5, topo
                # Usmm is tested at the same time in debugmode
                # Check if the optimization local_usmm and local_usmm_csx is
                # applied
                assert isinstance(topo[0].op,
                                  theano.sparse.basic.CSMProperties)
                assert isinstance(topo[1].op, theano.tensor.DimShuffle)
                assert isinstance(topo[2].op, theano.tensor.Subtensor)
                assert topo[3].op == theano.tensor.neg
                assert isinstance(topo[4].op, UsmmCscDense)
                if inplace:
                    assert topo[4].op.inplace
            elif not fast_compile:
                # The op Usmm should be inserted
                assert len(topo) == 3, topo
                assert isinstance(topo[0].op, theano.tensor.DimShuffle)
                assert topo[1].op == theano.tensor.neg
                assert isinstance(topo[2].op, theano.sparse.Usmm)

    def test_infer_shape(self):
        def mat(format, name, dtype):
            if format == 'dense':
                return theano.tensor.matrix(name, dtype=dtype)
            else:
                return theano.sparse.matrix(format, name, dtype=dtype)

        params = [('float32', 'float64', 'int16', 'complex64', 'csc', 'dense'),
                  ('float32', 'float64', 'int16', 'complex64', 'csr', 'dense')]
        for dtype1, dtype2, dtype3, dtype4, format1, format2 in params:
            if format1 == 'dense' and format2 == 'dense':
                # Usmm won't be used!
                continue
            x = mat(format1, 'x', dtype1)
            y = mat(format2, 'y', dtype2)
            a = theano.tensor.scalar('a', dtype=dtype3)
            z = theano.shared(np.asarray(self.z, dtype=dtype4).copy())

            f_b = lambda z, a, x, y: z - a * (x * y)
            x_data = np.asarray(self.x, dtype=dtype1)
            if format1 != 'dense':
                x_data = as_sparse_format(x_data, format1)
            y_data = np.asarray(self.y, dtype=dtype2)
            if format2 != 'dense':
                y_data = as_sparse_format(y_data, format2)
            a_data = np.asarray(1.5, dtype=dtype3)
            z_data = np.asarray(self.z, dtype=dtype4)

            f_b_out = f_b(z_data, a_data, x_data, y_data)

            # Can it work inplace?
            inplace = dtype4 == theano.scalar.upcast(dtype1, dtype2, dtype3)

            # To make it easier to check the toposort
            mode = theano.compile.mode.get_default_mode().excluding('fusion')

            # test infer_shape of Dot got applied
            f_shape = theano.function([a, x, y],
                                      (z - a * theano.sparse.dot(x, y)).shape,
                                      mode=mode)
            assert all(f_shape(a_data, x_data, y_data) == f_b_out.shape)
            topo = f_shape.maker.fgraph.toposort()
            if theano.config.mode != 'FAST_COMPILE':
                nb = 0
            else:
                nb = 1
            assert sum([isinstance(node.op, (Dot, Usmm, UsmmCscDense))
                        for node in topo]) == nb


class test_zeros_like(unittest.TestCase):
    def test(self):
        x = theano.sparse.csr_matrix()
        f = theano.function([x], theano.sparse.sp_zeros_like(x))
        vx = scipy.sparse.csr_matrix(np.asarray(
            np.random.binomial(1, 0.5, (100, 100)),
            dtype=theano.config.floatX))

        fx = f(vx)

        assert fx.nnz == 0
        assert fx.shape == vx.shape


def test_shape_i():
    sparse_dtype = 'float32'

    a = SparseType('csr', dtype=sparse_dtype)()
    f = theano.function([a], a.shape[1])
    assert f(sp.csr_matrix(random_lil((100, 10), sparse_dtype, 3))) == 10


def test_shape():
    # Test that getting the shape of a sparse variable
    # does not actually create a dense tensor in the process.
    sparse_dtype = 'float32'

    a = SparseType('csr', dtype=sparse_dtype)()
    f = theano.function([a], a.shape)
    assert np.all(f(sp.csr_matrix(random_lil((100, 10), sparse_dtype, 3)))
                     == (100, 10))
    if theano.config.mode != 'FAST_COMPILE':
        topo = f.maker.fgraph.toposort()
        assert len(topo) == 3
        assert isinstance(topo[0].op, tensor.opt.Shape_i)
        assert isinstance(topo[1].op, tensor.opt.Shape_i)
        assert isinstance(topo[2].op, tensor.opt.MakeVector)


def test_may_share_memory():
    a = scipy.sparse.csc_matrix(scipy.sparse.eye(5, 3))
    b = scipy.sparse.csc_matrix(scipy.sparse.eye(4, 3))
    as_ar = lambda a: theano._asarray(a, dtype='int32')
    for a_, b_, rep in [(a, a, True),
                        (b, b, True),
                        (a, b, False),
                        (a, a.data, True),
                        (a, a.indptr, True),
                        (a, a.indices, True),
                        (a, as_ar(a.shape), False),
                        (a.data, a, True),
                        (a.indptr, a, True),
                        (a.indices, a, True),
                        (as_ar(a.shape), a, False),
                        (b, b.data, True),
                        (b, b.indptr, True),
                        (b, b.indices, True),
                        (b, as_ar(b.shape), False),
                        (b.data, b, True),
                        (b.indptr, b, True),
                        (b.indices, b, True),
                        (as_ar(b.shape), b, False),
                        (b.data, a, False),
                        (b.indptr, a, False),
                        (b.indices, a, False),
                        (as_ar(b.shape), a, False),
                        (a.transpose(), a, True),
                        (b.transpose(), b, True),
                        (a.transpose(), b, False),
                        (b.transpose(), a, False),
                        ]:

        assert SparseType.may_share_memory(a_, b_) == rep


def test_sparse_shared_memory():
    # Note : There are no inplace ops on sparse matrix yet. If one is
    # someday implemented, we could test it here.
    a = random_lil((3, 4), 'float32', 3).tocsr()
    m1 = random_lil((4, 4), 'float32', 3).tocsr()
    m2 = random_lil((4, 4), 'float32', 3).tocsr()
    x = SparseType('csr', dtype='float32')()
    y = SparseType('csr', dtype='float32')()

    sdot = theano.sparse.structured_dot
    z = sdot(x * 3, m1) + sdot(y * 2, m2)

    f = theano.function([theano.In(x, mutable=True),
                         theano.In(y, mutable=True)], z, mode='FAST_RUN')

    def f_(x, y, m1=m1, m2=m2):
        return ((x * 3) * m1) + ((y * 2) * m2)

    assert SparseType.may_share_memory(a, a)  # This is trivial
    result = f(a, a)
    result_ = f_(a, a)
    assert (result_.todense() == result.todense()).all()


def test_size():
    """
    Ensure the `size` attribute of sparse matrices behaves as in numpy.
    """
    for sparse_type in ('csc_matrix', 'csr_matrix'):
        x = getattr(theano.sparse, sparse_type)()
        y = getattr(scipy.sparse, sparse_type)((5, 7)).astype(config.floatX)
        get_size = theano.function([x], x.size)

        def check():
            assert y.size == get_size(y)
        # We verify that the size is correctly updated as we store more data
        # into the sparse matrix (including zeros).
        check()
        y[0, 0] = 1
        check()
        y[0, 1] = 0
        check()


class ColScaleCSCTester(utt.InferShapeTester):
    def setUp(self):
        super(ColScaleCSCTester, self).setUp()
        self.op = sparse.col_scale

    def test_op(self):
        for format in sparse.sparse_formats:
            variable, data = sparse_random_inputs(format, shape=(8, 10))
            variable.append(tensor.vector())
            data.append(np.random.random(10).astype(config.floatX))

            f = theano.function(variable, self.op(*variable))

            tested = f(*data)
            x, s = data[0].toarray(), data[1][np.newaxis, :]
            expected = x * s

            assert tested.format == format
            utt.assert_allclose(expected, tested.toarray())

    def test_infer_shape(self):
        for format, cls in [('csc', sparse.ColScaleCSC),
                            ('csr', sparse.RowScaleCSC)]:
            variable, data = sparse_random_inputs(format, shape=(8, 10))
            variable.append(tensor.vector())
            data.append(np.random.random(10).astype(config.floatX))

            self._compile_and_check(variable,
                                    [self.op(*variable)],
                                    data,
                                    cls)

    def test_grad(self):
        for format in sparse.sparse_formats:
            variable, data = sparse_random_inputs(format, shape=(8, 10))
            variable.append(tensor.vector())
            data.append(np.random.random(10).astype(config.floatX))

            verify_grad_sparse(self.op, data, structured=True)


class RowScaleCSCTester(utt.InferShapeTester):
    def setUp(self):
        super(RowScaleCSCTester, self).setUp()
        self.op = sparse.row_scale

    def test_op(self):
        for format in sparse.sparse_formats:
            variable, data = sparse_random_inputs(format, shape=(8, 10))
            variable.append(tensor.vector())
            data.append(np.random.random(8).astype(config.floatX))

            f = theano.function(variable, self.op(*variable))

            tested = f(*data)
            x, s = data[0].toarray(), data[1][:, np.newaxis]
            expected = x * s

            assert tested.format == format
            utt.assert_allclose(expected, tested.toarray())

    def test_infer_shape(self):
        for format, cls in [('csc', sparse.RowScaleCSC),
                            ('csr', sparse.ColScaleCSC)]:
            variable, data = sparse_random_inputs(format, shape=(8, 10))
            variable.append(tensor.vector())
            data.append(np.random.random(8).astype(config.floatX))

            self._compile_and_check(variable,
                                    [self.op(*variable)],
                                    data,
                                    cls)

    def test_grad(self):
        for format in sparse.sparse_formats:
            variable, data = sparse_random_inputs(format, shape=(8, 10))
            variable.append(tensor.vector())
            data.append(np.random.random(8).astype(config.floatX))

            verify_grad_sparse(self.op, data, structured=True)


class SpSumTester(utt.InferShapeTester):
    possible_axis = [None, 0, 1]

    def setUp(self):
        super(SpSumTester, self).setUp()
        self.op_class = sparse.SpSum
        self.op = sparse.sp_sum

    def test_op(self):
        for format in sparse.sparse_formats:
            for axis in self.possible_axis:
                variable, data = sparse_random_inputs(format,
                                                      shape=(10, 10))

                z = theano.sparse.sp_sum(variable[0], axis=axis)
                if axis is None:
                    assert z.type.broadcastable == ()
                else:
                    assert z.type.broadcastable == (False, )

                f = theano.function(variable, self.op(variable[0], axis=axis))
                tested = f(*data)
                expected = data[0].todense().sum(axis).ravel()
                utt.assert_allclose(expected, tested)

    def test_infer_shape(self):
        for format in sparse.sparse_formats:
            for axis in self.possible_axis:
                variable, data = sparse_random_inputs(format,
                                                      shape=(9, 10))
                self._compile_and_check(variable,
                                        [self.op(variable[0], axis=axis)],
                                        data,
                                        self.op_class)

    def test_grad(self):
        for format in sparse.sparse_formats:
            for axis in self.possible_axis:
                for struct in [True, False]:
                    variable, data = sparse_random_inputs(format,
                                                          shape=(9, 10))
                    verify_grad_sparse(
                        self.op_class(axis=axis, sparse_grad=struct),
                        data,
                        structured=struct)


class DiagTester(utt.InferShapeTester):
    def setUp(self):
        super(DiagTester, self).setUp()
        self.op_class = Diag
        self.op = diag

    def test_op(self):
        for format in sparse.sparse_formats:
            variable, data = sparse_random_inputs(format,
                                                  shape=(10, 10))

            z = self.op(*variable)
            assert z.type.broadcastable == (False, )

            f = theano.function(variable, z)
            tested = f(*data)
            expected = data[0].toarray().diagonal()

            utt.assert_allclose(expected, tested)

    def test_infer_shape(self):
        for format in sparse.sparse_formats:
            variable, data = sparse_random_inputs(format,
                                                  shape=(10, 10))
            self._compile_and_check(variable,
                                    [self.op(*variable)],
                                    data,
                                    self.op_class,
                                    warn=False)

    def test_grad(self):
        for format in sparse.sparse_formats:
            variable, data = sparse_random_inputs(format,
                                                  shape=(10, 10))
            verify_grad_sparse(
                self.op,
                data,
                structured=False)


class SquareDiagonalTester(utt.InferShapeTester):
    def setUp(self):
        super(SquareDiagonalTester, self).setUp()
        self.op_class = SquareDiagonal
        self.op = square_diagonal

    def test_op(self):
        for format in sparse.sparse_formats:
            for size in range(5, 9):
                variable = [tensor.vector()]
                data = [np.random.random(size).astype(config.floatX)]

                f = theano.function(variable, self.op(*variable))
                tested = f(*data).toarray()

                expected = np.diag(*data)
                utt.assert_allclose(expected, tested)
                assert tested.dtype == expected.dtype
                assert tested.shape == expected.shape

    def test_infer_shape(self):
        for format in sparse.sparse_formats:
            for size in range(5, 9):
                variable = [tensor.vector()]
                data = [np.random.random(size).astype(config.floatX)]

                self._compile_and_check(variable,
                                        [self.op(*variable)],
                                        data,
                                        self.op_class)

    def test_grad(self):
        for format in sparse.sparse_formats:
            for size in range(5, 9):
                variable = [tensor.vector()]
                data = [np.random.random(size).astype(config.floatX)]

                verify_grad_sparse(
                    self.op,
                    data,
                    structured=False)


class EnsureSortedIndicesTester(utt.InferShapeTester):
    def setUp(self):
        super(EnsureSortedIndicesTester, self).setUp()
        self.op_class = EnsureSortedIndices
        self.op = ensure_sorted_indices

    def test_op(self):
        for format in sparse.sparse_formats:
            for shape in zip(range(5, 9), range(3, 7)[::-1]):
                variable, data = sparse_random_inputs(format, shape=shape)

                f = theano.function(variable, self.op(*variable))
                tested = f(*data).toarray()
                expected = data[0].sorted_indices().toarray()

                utt.assert_allclose(expected, tested)

    def test_infer_shape(self):
        for format in sparse.sparse_formats:
            for shape in zip(range(5, 9), range(3, 7)[::-1]):
                variable, data = sparse_random_inputs(format, shape=shape)
                self._compile_and_check(variable,
                                        [self.op(*variable)],
                                        data,
                                        self.op_class)

    def test_grad(self):
        for format in sparse.sparse_formats:
            for shape in zip(range(5, 9), range(3, 7)[::-1]):
                variable, data = sparse_random_inputs(format, shape=shape)
                verify_grad_sparse(
                    self.op,
                    data,
                    structured=False)


class CleanTester(utt.InferShapeTester):
    def setUp(self):
        super(CleanTester, self).setUp()
        self.op = clean

    def test_op(self):
        for format in sparse.sparse_formats:
            for shape in zip(range(5, 9), range(3, 7)[::-1]):
                variable, data = sparse_random_inputs(format, shape=shape)

                data[0][0, 0] = data[0][1, 1] = 0

                f = theano.function(variable, self.op(*variable))
                tested = f(*data)
                expected = data[0]
                expected.eliminate_zeros()

                assert all(tested.data == expected.data)
                assert not all(tested.data == 0)

                tested = tested.toarray()
                expected = expected.toarray()
                utt.assert_allclose(expected, tested)

    def test_grad(self):
        for format in sparse.sparse_formats:
            for shape in zip(range(5, 9), range(3, 7)[::-1]):
                variable, data = sparse_random_inputs(format, shape=shape)
                verify_grad_sparse(
                    self.op,
                    data,
                    structured=False)


class Remove0Tester(utt.InferShapeTester):
    def setUp(self):
        super(Remove0Tester, self).setUp()
        self.op_class = Remove0

    def test_remove0(self):
        configs = [
            # structure type, numpy matching class
            ('csc', scipy.sparse.csc_matrix),
            ('csr', scipy.sparse.csr_matrix), ]

        for format, matrix_class in configs:
            for zero, unsor in [(True, True), (True, False),
                              (False, True), (False, False)]:
                (x,), (mat,) = sparse_random_inputs(format, (6, 8),
                                            out_dtype=config.floatX,
                                            explicit_zero=zero,
                                            unsorted_indices=unsor)
                assert 0 in mat.data or not zero
                assert not mat.has_sorted_indices or not unsor

                # the In thingy has to be there because theano has as rule not
                # to optimize inputs
                f = theano.function([theano.In(x, borrow=True, mutable=True)],
                                    Remove0()(x))

                # assert optimization local_inplace_remove0 is applied in
                # modes with optimization
                if theano.config.mode not in ['FAST_COMPILE']:
                    # list of apply nodes in the optimized graph.
                    nodes = f.maker.fgraph.toposort()
                    # Check there isn't any Remove0 instance not inplace.
                    assert not any([isinstance(node.op, Remove0) and
                                    not node.op.inplace for node in nodes]), (
                           'Inplace optimization should have been applied')
                    # Check there is at least one Remove0 inplace.
                    assert any([isinstance(node.op, Remove0) and node.op.inplace
                                for node in nodes])
                # checking
                # makes sense to change its name
                target = mat
                result = f(mat)
                mat.eliminate_zeros()
                msg = 'Matrices sizes differ. Have zeros been removed ?'
                assert result.size == target.size, msg
                if unsor:
                    assert not result.has_sorted_indices
                    assert not target.has_sorted_indices
                else:
                    assert result.has_sorted_indices
                    assert target.has_sorted_indices

    def test_infer_shape(self):
        mat = (np.arange(12) + 1).reshape((4, 3))
        mat[0, 1] = mat[1, 0] = mat[2, 2] = 0

        x_csc = theano.sparse.csc_matrix(dtype=theano.config.floatX)
        mat_csc = sp.csc_matrix(mat, dtype=theano.config.floatX)
        self._compile_and_check([x_csc],
                                [Remove0()(x_csc)],
                                [mat_csc],
                                self.op_class)

        x_csr = theano.sparse.csr_matrix(dtype=theano.config.floatX)
        mat_csr = sp.csr_matrix(mat, dtype=theano.config.floatX)
        self._compile_and_check([x_csr],
                                [Remove0()(x_csr)],
                                [mat_csr],
                                self.op_class)

    def test_grad(self):
        mat = (np.arange(9) + 1).reshape((3, 3))
        mat[0, 1] = mat[1, 0] = mat[2, 2] = 0

        mat_csc = sp.csc_matrix(mat, dtype=theano.config.floatX)
        verify_grad_sparse(Remove0(), [mat_csc])

        mat_csr = sp.csr_matrix(mat, dtype=theano.config.floatX)
        verify_grad_sparse(Remove0(), [mat_csr])


class Test_getitem(unittest.TestCase):
    def setUp(self):
        self.rng = np.random.RandomState(utt.fetch_seed())

    def test_GetItemList(self):

        a, A = sparse_random_inputs('csr', (4, 5))
        b, B = sparse_random_inputs('csc', (4, 5))
        y = a[0][[0, 1, 2, 3, 1]]
        z = b[0][[0, 1, 2, 3, 1]]

        fa = theano.function([a[0]], y)
        fb = theano.function([b[0]], z)

        t_geta = fa(A[0]).todense()
        t_getb = fb(B[0]).todense()

        s_geta = scipy.sparse.csr_matrix(A[0])[[0, 1, 2, 3, 1]].todense()
        s_getb = scipy.sparse.csc_matrix(B[0])[[0, 1, 2, 3, 1]].todense()

        utt.assert_allclose(t_geta, s_geta)
        utt.assert_allclose(t_getb, s_getb)

    def test_GetItemList_wrong_index(self):
        a, A = sparse_random_inputs('csr', (4, 5))
        y = a[0][[0, 4]]
        f = theano.function([a[0]], y)

        self.assertRaises(IndexError, f, A[0])

    def test_get_item_list_grad(self):
        op = theano.sparse.basic.GetItemList()
        def op_with_fixed_index(x):
            return op(x, index=np.asarray([0, 1]))

        x, x_val = sparse_random_inputs("csr", (4, 5))

        try:
            verify_grad_sparse(op_with_fixed_index, x_val)
        except NotImplementedError as e:
            assert "Scipy version is to old" in str(e)

    def test_GetItem2Lists(self):

        a, A = sparse_random_inputs('csr', (4, 5))
        b, B = sparse_random_inputs('csc', (4, 5))
        y = a[0][[0, 0, 1, 3], [0, 1, 2, 4]]
        z = b[0][[0, 0, 1, 3], [0, 1, 2, 4]]

        fa = theano.function([a[0]], y)
        fb = theano.function([b[0]], z)

        t_geta = fa(A[0])
        t_getb = fb(B[0])

        s_geta = np.asarray(scipy.sparse.csr_matrix(A[0])[[0, 0, 1, 3], [0, 1, 2, 4]])
        s_getb = np.asarray(scipy.sparse.csc_matrix(B[0])[[0, 0, 1, 3], [0, 1, 2, 4]])

        utt.assert_allclose(t_geta, s_geta)
        utt.assert_allclose(t_getb, s_getb)

    def test_GetItem2Lists_wrong_index(self):
        a, A = sparse_random_inputs('csr', (4, 5))
        y1 = a[0][[0, 5], [0, 3]]
        y2 = a[0][[0, 3], [0, 5]]

        f1 = theano.function([a[0]], y1)
        f2 = theano.function([a[0]], y2)

        self.assertRaises(IndexError, f1, A[0])
        self.assertRaises(IndexError, f2, A[0])

    def test_get_item_2lists_grad(self):
        op = theano.sparse.basic.GetItem2Lists()
        def op_with_fixed_index(x):
            return op(x, ind1=np.asarray([0, 1]), ind2=np.asarray([2, 3]))

        x, x_val = sparse_random_inputs("csr", (4, 5))

        verify_grad_sparse(op_with_fixed_index, x_val)

    def test_GetItem2D(self):
        scipy_ver = [int(n) for n in scipy.__version__.split('.')[:2]]
        is_supported_version = bool(scipy_ver >= [0, 14])

        sparse_formats = ('csc', 'csr')
        for format in sparse_formats:
            x = theano.sparse.matrix(format, name='x')
            a = theano.tensor.iscalar('a')
            b = theano.tensor.iscalar('b')
            c = theano.tensor.iscalar('c')
            d = theano.tensor.iscalar('d')
            e = theano.tensor.iscalar('e')
            f = theano.tensor.iscalar('f')

            # index
            m = 1
            n = 5
            p = 10
            q = 15
            if is_supported_version:
                j = 2
                k = 3
            else:
                j = None
                k = None

            vx = as_sparse_format(self.rng.binomial(1, 0.5, (100, 97)),
                                      format).astype(theano.config.floatX)

            #mode_no_debug = theano.compile.mode.get_default_mode()
            # if isinstance(mode_no_debug, theano.compile.DebugMode):
            #    mode_no_debug = 'FAST_RUN'
            if is_supported_version:
                f1 = theano.function([x, a, b, c, d, e, f], x[a:b:e, c:d:f])
                r1 = f1(vx, m, n, p, q, j, k)
                t1 = vx[m:n:j, p:q:k]
            else:
                f1 = theano.function([x, a, b, c, d], x[a:b, c:d])
                r1 = f1(vx, m, n, p, q)
                t1 = vx[m:n, p:q]
            assert r1.shape == t1.shape
            assert np.all(t1.toarray() == r1.toarray())

            """
            Important: based on a discussion with both Fred and James
            The following indexing methods is not supported because the rval
            would be a sparse matrix rather than a sparse vector, which is a
            deviation from numpy indexing rule. This decision is made largely
            for keeping the consistency between numpy and theano.

            f2 = theano.function([x, a, b, c], x[a:b, c])
            r2 = f2(vx, m, n, p)
            t2 = vx[m:n, p]
            assert r2.shape == t2.shape
            assert np.all(t2.toarray() == r2.toarray())

            f3 = theano.function([x, a, b, c], x[a, b:c])
            r3 = f3(vx, m, n, p)
            t3 = vx[m, n:p]
            assert r3.shape == t3.shape
            assert np.all(t3.toarray() == r3.toarray())

            f5 = theano.function([x], x[1:2,3])
            r5 = f5(vx)
            t5 = vx[1:2, 3]
            assert r5.shape == t5.shape
            assert np.all(r5.toarray() == t5.toarray())

            f7 = theano.function([x], x[50])
            r7 = f7(vx)
            t7 = vx[50]
            assert r7.shape == t7.shape
            assert np.all(r7.toarray() == t7.toarray())
            """
            if is_supported_version:
                f4 = theano.function([x, a, b, e], x[a:b:e])
                r4 = f4(vx, m, n, j)
                t4 = vx[m:n:j]
            else:
                f4 = theano.function([x, a, b], x[a:b])
                r4 = f4(vx, m, n)
                t4 = vx[m:n]
            assert r4.shape == t4.shape
            assert np.all(t4.toarray() == r4.toarray())

            #-----------------------------------------------------------
            # test cases using int indexing instead of theano variable
            f6 = theano.function([x], x[1:10:j, 10:20:k])
            r6 = f6(vx)
            t6 = vx[1:10:j, 10:20:k]
            assert r6.shape == t6.shape
            assert np.all(r6.toarray() == t6.toarray())

            #----------------------------------------------------------
            # test cases with indexing both with theano variable and int
            if is_supported_version:
                f8 = theano.function([x, a, b, e], x[a:b:e, 10:20:1])
                r8 = f8(vx, m, n, j)
                t8 = vx[m:n:j, 10:20:1]
            else:
                f8 = theano.function([x, a, b], x[a:b, 10:20])
                r8 = f8(vx, m, n)
                t8 = vx[m:n, 10:20]
            assert r8.shape == t8.shape
            assert np.all(r8.toarray() == t8.toarray())

            f9 = theano.function([x, a, b], x[1:a:j, 1:b:k])
            r9 = f9(vx, p, q)
            t9 = vx[1:p:j, 1:q:k]
            assert r9.shape == t9.shape
            assert np.all(r9.toarray() == t9.toarray())

            #-----------------------------------------------------------
            # Test mixing None and variables
            f10 = theano.function([x, a, b], x[:a, :b])
            r10 = f10(vx, p, q)
            t10 = vx[:p, :q]
            assert r10.shape == t10.shape
            assert np.all(r10.toarray() == t10.toarray())

            f11 = theano.function([x, a], x[:, a:])
            r11 = f11(vx, p)
            t11 = vx[:, p:]
            assert r11.shape == t11.shape
            assert np.all(r11.toarray() == t11.toarray())

            # Test that is work with shared variable
            sx = theano.shared(vx)
            f12 = theano.function([a], sx[:, a:])
            r12 = f12(p)
            t12 = vx[:, p:]
            assert r12.shape == t12.shape
            assert np.all(r12.toarray() == t12.toarray())

            #------------------------------------------------------------
            # Invalid things
            # The syntax is a bit awkward because assertRaises forbids
            # the [] shortcut for getitem.
            # x[a:b] is not accepted because we don't have sparse vectors
            self.assertRaises(NotImplementedError,
                              x.__getitem__, (slice(a, b), c))

            # x[a:b:step, c:d] is not accepted because scipy silently drops
            # the step (!)
            if not is_supported_version:
                self.assertRaises(ValueError,
                                  x.__getitem__, (slice(a, b, -1), slice(c, d)))
                self.assertRaises(ValueError,
                                  x.__getitem__, (slice(a, b), slice(c, d, 2)))
            else:
                raise SkipTest("Slicing with step is supported.")

            # Advanced indexing is not supported
            self.assertRaises(ValueError,
                              x.__getitem__,
                              (tensor.ivector('l'), slice(a, b)))

            # Indexing with random things is not supported either
            self.assertRaises(ValueError,
                              x.__getitem__, slice(tensor.fscalar('f'), None))
            self.assertRaises(ValueError,
                              x.__getitem__,
                              (slice(None), slice([1, 3, 4], None)))

    def test_GetItemScalar(self):
        sparse_formats = ('csc', 'csr')
        for format in sparse_formats:
            x = theano.sparse.csc_matrix('x')
            a = theano.tensor.iscalar()
            b = theano.tensor.iscalar()

            m = 50
            n = 42

            vx = as_sparse_format(self.rng.binomial(1, 0.5, (97, 100)),
                                  format).astype(theano.config.floatX)

            f1 = theano.function([x, a, b], x[a, b])
            r1 = f1(vx, 10, 10)
            t1 = vx[10, 10]
            assert r1.shape == t1.shape
            assert np.all(t1 == r1)

            f2 = theano.function([x, a], x[50, a])
            r2 = f2(vx, m)
            t2 = vx[50, m]
            assert r2.shape == t2.shape
            assert np.all(t2 == r2)

            f3 = theano.function([x, a], x[a, 50])
            r3 = f3(vx, m)
            t3 = vx[m, 50]
            assert r3.shape == t3.shape
            assert np.all(t3 == r3)

            f4 = theano.function([x], x[50, 42])
            r4 = f4(vx)
            t4 = vx[m, n]
            assert r3.shape == t3.shape
            assert np.all(t4 == r4)

            # Test that is work with shared variable
            sx = theano.shared(vx)
            f1 = theano.function([a, b], sx[a, b])
            r1 = f1(10, 10)
            t1 = vx[10, 10]
            assert r1.shape == t1.shape
            assert np.all(t1 == r1)


class CastTester(utt.InferShapeTester):
    def setUp(self):
        super(CastTester, self).setUp()

    # slow but only test
    def test_cast(self):
        for format in sparse.sparse_formats:
            for i_dtype in sparse.all_dtypes:
                for o_dtype in sparse.all_dtypes:
                    (variable, ),  (data, ) = sparse_random_inputs(
                        format,
                        shape=(4, 7),
                        out_dtype=i_dtype)

                    func = theano.function([variable], cast(variable, o_dtype))
                    cls = theano.function([variable], Cast(o_dtype)(variable))
                    prop = theano.function([variable],
                                           variable.astype(o_dtype))

                    t_func, t_cls, t_prop = func(data), cls(data), prop(data)

                    expected = data.toarray().astype(o_dtype)

                    assert t_func.format == format
                    assert t_cls.format == format
                    assert t_prop.format == format

                    t_func = t_func.toarray()
                    t_cls = t_cls.toarray()
                    t_prop = t_prop.toarray()

                    utt.assert_allclose(expected, t_func)
                    utt.assert_allclose(expected, t_cls)
                    utt.assert_allclose(expected, t_prop)

    @attr('slow')
    def test_infer_shape(self):
        for format in sparse.sparse_formats:
            for i_dtype in sparse.all_dtypes:
                for o_dtype in sparse.all_dtypes:
                    variable, data = sparse_random_inputs(
                        format,
                        shape=(4, 7),
                        out_dtype=i_dtype)
                    self._compile_and_check(variable,
                                            [Cast(o_dtype)(*variable)],
                                            data,
                                            Cast)

    def test_grad(self):
        for format in sparse.sparse_formats:
            for i_dtype in sparse.float_dtypes:
                for o_dtype in tensor.float_dtypes:
                    if o_dtype == 'float16':
                        # Don't test float16 output.
                        continue
                    _, data = sparse_random_inputs(
                        format,
                        shape=(4, 7),
                        out_dtype=i_dtype)

                    eps = None
                    if o_dtype == 'float32':
                        eps = 1e-2

                    verify_grad_sparse(Cast(o_dtype), data, eps=eps)


def _format_info(nb):
    x = {}
    mat = {}

    for format in sparse.sparse_formats:
        variable = getattr(theano.sparse, format + '_matrix')
        spa = getattr(sp, format + '_matrix')

        x[format] = [variable() for t in range(nb)]
        mat[format] = [spa(random_lil((3, 4), theano.config.floatX, 8))
                       for t in range(nb)]
    return x, mat


class _HVStackTester(utt.InferShapeTester):
    """Test for both HStack and VStack.

    """
    nb = 3  # Number of sparse matrix to stack
    x, mat = _format_info(nb)

    def test_op(self):
        for format in sparse.sparse_formats:
            for out_f in sparse.sparse_formats:
                for dtype in sparse.all_dtypes:
                    blocks = self.mat[format]

                    f = theano.function(
                        self.x[format],
                        self.op_class(
                            format=out_f, dtype=dtype)(*self.x[format]),
                        allow_input_downcast=True)

                    tested = f(*blocks)
                    expected = self.expected_f(blocks,
                                               format=out_f,
                                               dtype=dtype)

                    utt.assert_allclose(expected.toarray(), tested.toarray())
                    assert tested.format == expected.format
                    assert tested.dtype == expected.dtype

    def test_infer_shape(self):
        for format in sparse.sparse_formats:
            self._compile_and_check(self.x[format],
                                    [self.op_class(dtype='float64')
                                     (*self.x[format])],
                                    self.mat[format],
                                    self.op_class)

    def test_grad(self):
        for format in sparse.sparse_formats:
            for out_f in sparse.sparse_formats:
                for dtype in sparse.float_dtypes:
                    verify_grad_sparse(
                        self.op_class(format=out_f, dtype=dtype),
                        self.mat[format],
                        structured=False,
                        eps=1e-2,
                        )


def _hv_switch(op, expected_function):
    """Return the right test class for HStack or VStack.

    :Parameters:
    - `op`: HStack or VStack class.
    - `expected_function`: function from scipy for comparaison.
    """

    class XStackTester(_HVStackTester):
        op_class = op

        def expected_f(self, a, format=None, dtype=None):
            return expected_function(a, format, dtype)
    XStackTester.__name__ = op.__name__ + "Tester"
    if hasattr(XStackTester, '__qualname__'):
        XStackTester.__qualname__ = XStackTester.__name__
    return XStackTester

HStackTester = _hv_switch(HStack, sp.hstack)
VStackTester = _hv_switch(VStack, sp.vstack)


class AddSSDataTester(utt.InferShapeTester):
    x = {}
    a = {}

    def setUp(self):
        super(AddSSDataTester, self).setUp()
        self.op_class = AddSSData

        for format in sparse.sparse_formats:
            variable = getattr(theano.sparse, format + '_matrix')

            rand = np.array(
                np.random.randint(1, 4, size=(3, 4)) - 1,
                dtype=theano.config.floatX)
            constant = as_sparse_format(rand, format)

            self.x[format] = [variable() for t in range(2)]
            self.a[format] = [constant for t in range(2)]

    def test_op(self):
        for format in sparse.sparse_formats:
            f = theano.function(
                self.x[format],
                add_s_s_data(*self.x[format]))

            tested = f(*self.a[format])
            expected = 2 * self.a[format][0]

            utt.assert_allclose(expected.toarray(), tested.toarray())
            assert tested.format == expected.format
            assert tested.dtype == expected.dtype

    def test_infer_shape(self):
        for format in sparse.sparse_formats:
            self._compile_and_check(self.x[format],
                                    [add_s_s_data(*self.x[format])],
                                    self.a[format],
                                    self.op_class)

    def test_grad(self):
        for format in sparse.sparse_formats:
            verify_grad_sparse(self.op_class(),
                               self.a[format],
                               structured=True)


def elemwise_checker(op, expected_f, gap=None, test_dtypes=None,
                     grad_test=True, name=None, gap_grad=None):
    """Return the appropriate test class for the elemwise on sparse.

    :param op: Op to test.
    :expected_f: Function use to compare. This function must act
                 on dense matrix. If the op is structured
                 see the `structure_function` decorator to make
                 this function structured.
    :param gap: Tuple for the range of the random sample. When
                length is 1, it is assumed to be the exclusive
                max, when `gap` = (`a`, `b`) it provide a sample
                from [a, b[. If `None` is used, it provide [0, 1]
                for float dtypes and [0, 50[ for integer dtypes.
    :param test_dtypes: Particular dtypes for testing the op.
                        If `None`, this is set to the most common
                        dtypes.
    :param grad_test: True for testing the grad. False will
                      skip this test.
    :param gap_grad: If None, we reuse gap. Otherwise it is the same as gap
                     but for testing the gradiant of the op.

    :return: The class that perform the tests, not an instance
             of the class.
    """

    if test_dtypes is None:
        test_dtypes = sparse.all_dtypes

    class Tester(unittest.TestCase):

        def setUp(self):
            super(Tester, self).setUp()
            self.op = op
            self.expected_f = expected_f
            self.gap = gap
            if gap_grad is not None:
                self.gap_grad = gap_grad
            else:
                self.gap_grad = gap
            # Ensure the test's name is correct.
            utt.seed_rng()
            assert eval(self.__class__.__name__) is self.__class__

        def test_op(self):
            for format in sparse.sparse_formats:
                for dtype in test_dtypes:
                    if dtype == 'int8' or dtype == 'uint8':
                        continue

                    # When testing with unsigned integers,
                    # we must check if the gap contains
                    # negative numbers.
                    if dtype.startswith('uint'):
                        if self.gap and len(self.gap) == 2 and self.gap[0] < 0:
                            if self.gap[1] >= 1:
                                self.gap = (0, self.gap[1])
                            else:
                                raise TypeError('Gap not suitable for',
                                                dtype, self.__name__)

                    variable, data = sparse_random_inputs(
                        format,
                        shape=(4, 7),
                        out_dtype=dtype,
                        gap=self.gap)

                    f = theano.function(variable, self.op(*variable))

                    tested = f(*data)
                    data = [m.toarray() for m in data]
                    expected = self.expected_f(*data)

                    assert tested.format == format
                    tested = tested.toarray()

                    try:
                        utt.assert_allclose(expected, tested)
                    except AssertionError:
                        raise AssertionError(self.__name__)

                # Test with int8 as dtype
                # These tests are not in the loop for two reasons.
                # First, in recent version of numpy, when a numpy
                # function have int8 as input dtype, it returns a
                # float16 as output dtype. Since this does not provide
                # enough precision, we upcast the data before we apply the
                # function.
                # Second, the tolerance for the checkup in DebugMode
                # is too high.
                for dtype in ['int8', 'uint8']:
                    if dtype in test_dtypes:
                        if self.gap:
                            domain = self.gap
                            # When testing with unsigned integers,
                            # we must check if the gap contains
                            # negative numbers.
                            if dtype == 'uint8':
                                if len(domain) == 2 and domain[0] < 0:
                                    if domain[1] >= 1:
                                        domain = (0, domain[1])
                                    else:
                                        raise TypeError('Gap not suitable for',
                                                        dtype, self.__name__)

                        else:
                            domain = (0, 5)

                        variable, data = sparse_random_inputs(
                            format,
                            shape=(4, 7),
                            out_dtype=dtype,
                            gap=domain)

                        f = theano.function(variable, self.op(*variable))

                        old_value = (tensor.basic.float32_atol,
                                     tensor.basic.float32_rtol,
                                     tensor.basic.float64_atol,
                                     tensor.basic.float64_rtol)
                        tensor.basic.float32_atol = 1e-4
                        tensor.basic.float32_rtol = 1e-3
                        tensor.basic.float64_atol = 1e-3
                        tensor.basic.float64_rtol = 1e-4
                        try:
                            tested = f(*data)
                        finally:
                            (tensor.basic.float32_atol,
                             tensor.basic.float32_rtol,
                             tensor.basic.float64_atol,
                             tensor.basic.float64_rtol) = old_value

                        data = [m.toarray().astype('float32') for m in data]
                        expected = self.expected_f(*data)

                        assert tested.format == format
                        tested = tested.toarray()

                        try:
                            utt.assert_allclose(tested, expected, rtol=1e-2)
                        except AssertionError:
                            raise AssertionError(self.__name__)

        if grad_test:
            def test_grad(self):
                for format in sparse.sparse_formats:
                    for dtype in sparse.float_dtypes:
                        variable, data = sparse_random_inputs(
                            format,
                            shape=(4, 7),
                            out_dtype=dtype,
                            gap=self.gap_grad)

                        verify_grad_sparse(self.op,
                                           data,
                                           structured=True)

    # Set proper class name to uniquely identify tests.
    # Note that it is important to run this code *outside* of the `Tester`
    # class itself, otherwise it will not work properly for some reason.
    if name is None:
        name = op.__name__.capitalize() + 'Tester'
    Tester.__name__ = name
    if hasattr(Tester, '__qualname__'):
        Tester.__qualname__ = name
    assert 'Roundhalftoeven' not in Tester.__name__

    return Tester


def test_hstack_vstack():
    """
    Tests sparse.hstack and sparse.vstack (as opposed to the HStack and VStack
    classes that they wrap).
    """

    def make_block(dtype):
        return theano.sparse.csr_matrix(name="%s block" % dtype,
                                        dtype=dtype)

    def get_expected_dtype(blocks, to_dtype):
        if to_dtype is None:
            block_dtypes = tuple(b.dtype for b in blocks)
            return theano.scalar.upcast(*block_dtypes)
        else:
            return to_dtype

    # a deliberately weird mix of dtypes to stack
    dtypes = ('complex128', theano.config.floatX)

    blocks = [make_block(dtype) for dtype in dtypes]

    for stack_dimension, stack_function in enumerate((theano.sparse.vstack,
                                                      theano.sparse.hstack)):

        for to_dtype in (None, ) + dtypes:
            stacked_blocks = stack_function(blocks, dtype=to_dtype)
            expected_dtype = get_expected_dtype(blocks, to_dtype)
            assert stacked_blocks.dtype == expected_dtype


def structure_function(f, index=0):
    """Decorator to structure a function wich
    apply on dense matrix.

    Here, the inputs of the function must be
    dense matrix. The sparse pattern is
    determined by finding the zeros.

    :param index: The index of the parameter
                  from wich the function must
                  be structured.

    :return: The structured function for its
             `index` parameter.
    """

    def structured_function(*args):
        pattern = args[index]
        evaluated = f(*args)
        evaluated[pattern == 0] = 0
        return evaluated
    return structured_function

StructuredSigmoidTester = elemwise_checker(
    sparse.structured_sigmoid,
    structure_function(lambda x: 1.0 / (1.0 + np.exp(-x))),
    test_dtypes=[m for m in sparse.all_dtypes
                 if (not m in sparse.complex_dtypes and
                     not m.startswith('uint'))],
    gap=(-5, 5),
    name='StructuredSigmoidTester')

StructuredExpTester = elemwise_checker(
    sparse.structured_exp,
    structure_function(np.exp),
    name='StructuredExpTester')

StructuredLogTester = elemwise_checker(
    sparse.structured_log,
    structure_function(np.log),
    gap=(0.5, 10),
    name='StructuredLogTester')

StructuredPowTester = elemwise_checker(
    lambda x: sparse.structured_pow(x, 2),
    structure_function(lambda x: np.power(x, 2)),
    name='StructuredPowTester')

StructuredMinimumTester = elemwise_checker(
    lambda x: structured_minimum(x, 2),
    structure_function(lambda x: np.minimum(x, 2)),
    name='StructuredMinimumTester')

StructuredMaximumTester = elemwise_checker(
    lambda x: structured_maximum(x, 2),
    structure_function(lambda x: np.maximum(x, 2)),
    name='StructuredMaximumTester')

StructuredAddTester = elemwise_checker(
    lambda x: structured_add(x, 2),
    structure_function(lambda x: np.add(x, 2)),
    name='StructuredAddTester')

SinTester = elemwise_checker(
    sparse.sin,
    np.sin)

TanTester = elemwise_checker(
    sparse.tan,
    np.tan,
    gap=(-1, 1))

ArcsinTester = elemwise_checker(
    sparse.arcsin,
    np.arcsin,
    gap=(-1, 1),
    gap_grad=(-0.99, 0.99))

ArctanTester = elemwise_checker(
    sparse.arctan,
    np.arctan)

SinhTester = elemwise_checker(
    sparse.sinh,
    np.sinh)

ArcsinhTester = elemwise_checker(
    sparse.arcsinh,
    np.arcsinh,
    gap=(-1, 1))

TanhTester = elemwise_checker(
    sparse.tanh,
    np.tanh,
    gap=(-1, 1))

ArctanhTester = elemwise_checker(
    sparse.arctanh,
    np.arctanh,
    gap=(-0.9, 1),
    gap_grad=(-0.9, 0.95))

RintTester = elemwise_checker(
    sparse.rint,
    np.rint,
    grad_test=False,
    test_dtypes=sparse.float_dtypes)

SgnTester = elemwise_checker(
    sparse.sgn,
    np.sign,
    grad_test=False,
    test_dtypes=[m for m in sparse.all_dtypes
                 if (not m in sparse.complex_dtypes and
                     not m.startswith('uint'))])

CeilTester = elemwise_checker(
    sparse.ceil,
    np.ceil,
    grad_test=False,
    test_dtypes=[m for m in sparse.all_dtypes
                 if not m in sparse.complex_dtypes])

FloorTester = elemwise_checker(
    sparse.floor,
    np.floor,
    grad_test=False,
    test_dtypes=[m for m in sparse.all_dtypes
                 if not m in sparse.complex_dtypes])

Log1pTester = elemwise_checker(
    sparse.log1p,
    np.log1p,
    gap=(0.5, 10))

Expm1Tester = elemwise_checker(
    sparse.expm1,
    np.expm1)

Deg2radTester = elemwise_checker(
    sparse.deg2rad,
    np.deg2rad,
    test_dtypes=[m for m in sparse.all_dtypes
                 if not m in sparse.complex_dtypes])

Rad2degTester = elemwise_checker(
    sparse.rad2deg,
    np.rad2deg,
    test_dtypes=[m for m in sparse.all_dtypes
                 if not m in sparse.complex_dtypes])


TruncTester = elemwise_checker(
    sparse.trunc,
    np.trunc,
    test_dtypes=[m for m in sparse.all_dtypes
                 if not m in sparse.complex_dtypes])


SqrTester = elemwise_checker(
    sparse.sqr,
    lambda x: x * x)

SqrtTester = elemwise_checker(
    sparse.sqrt,
    np.sqrt,
    gap=(0, 10))

ConjTester = elemwise_checker(
    sparse.conj,
    np.conj,
    grad_test=False)


class MulSVTester(unittest.TestCase):
    def setUp(self):
        utt.seed_rng()

    def test_mul_s_v_grad(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csr', 'csc']:
            for dtype in ['float32', 'float64']:
                spmat = sp_types[format](random_lil((4, 3), dtype, 3))
                mat = np.asarray(np.random.rand(3), dtype=dtype)

                verify_grad_sparse(mul_s_v,
                                   [spmat, mat],
                                   structured=True)

    def test_mul_s_v(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csr', 'csc']:
            for dtype in ['float32', 'float64']:
                x = theano.sparse.SparseType(format, dtype=dtype)()
                y = tensor.vector(dtype=dtype)
                f = theano.function([x, y], mul_s_v(x, y))

                spmat = sp_types[format](random_lil((4, 3), dtype, 3))
                mat = np.asarray(np.random.rand(3), dtype=dtype)

                out = f(spmat, mat)

                utt.assert_allclose(spmat.toarray() * mat, out.toarray())


class StructuredAddSVTester(unittest.TestCase):
    def setUp(self):
        utt.seed_rng()

    def test_structured_add_s_v_grad(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csr', 'csc']:
            for dtype in ['float32', 'float64']:
                spmat = sp_types[format](random_lil((4, 3), dtype, 3))
                mat = np.asarray(np.random.rand(3), dtype=dtype)

                verify_grad_sparse(structured_add_s_v,
                                   [spmat, mat],
                                   structured=True)

    def test_structured_add_s_v(self):
        sp_types = {'csc': sp.csc_matrix,
                    'csr': sp.csr_matrix}

        for format in ['csr', 'csc']:
            for dtype in ['float32', 'float64']:
                x = theano.sparse.SparseType(format, dtype=dtype)()
                y = tensor.vector(dtype=dtype)
                f = theano.function([x, y], structured_add_s_v(x, y))

                spmat = sp_types[format](random_lil((4, 3), dtype, 3))
                spones = spmat.copy()
                spones.data = np.ones_like(spones.data)
                mat = np.asarray(np.random.rand(3), dtype=dtype)

                out = f(spmat, mat)

                utt.assert_allclose(as_ndarray(spones.multiply(spmat + mat)),
                                    out.toarray())


class TrueDotTester(utt.InferShapeTester):
    def setUp(self):
        super(TrueDotTester, self).setUp()
        self.op = true_dot
        self.op_class = TrueDot

    def test_op_ss(self):
        for format in sparse.sparse_formats:
            for dtype in sparse.all_dtypes:
                variable, data = sparse_random_inputs(format,
                                                      shape=(10, 10),
                                                      out_dtype=dtype,
                                                      n=2,
                                                      p=0.1)

                f = theano.function(variable, self.op(*variable))

                tested = f(*data)

                x, y = [m.toarray() for m in data]
                expected = np.dot(x, y)

                assert tested.format == format
                assert tested.dtype == expected.dtype
                tested = tested.toarray()
                utt.assert_allclose(tested, expected)

    def test_op_sd(self):
        for format in sparse.sparse_formats:
            for dtype in sparse.all_dtypes:
                variable, data = sparse_random_inputs(format,
                                                      shape=(10, 10),
                                                      out_dtype=dtype,
                                                      n=2,
                                                      p=0.1)
                variable[1] = tensor.TensorType(dtype=dtype,
                                                broadcastable=(False, False))()
                data[1] = data[1].toarray()

                f = theano.function(variable, self.op(*variable))

                tested = f(*data)
                expected = np.dot(data[0].toarray(), data[1])

                assert tested.format == format
                assert tested.dtype == expected.dtype
                tested = tested.toarray()
                utt.assert_allclose(tested, expected)

    def test_infer_shape(self):
        for format in sparse.sparse_formats:
            for dtype in sparse.all_dtypes:
                (x, ), (x_value, ) = sparse_random_inputs(format,
                                                          shape=(9, 10),
                                                          out_dtype=dtype,
                                                          p=0.1)
                (y, ), (y_value, ) = sparse_random_inputs(format,
                                                          shape=(10, 24),
                                                          out_dtype=dtype,
                                                          p=0.1)
                variable = [x, y]
                data = [x_value, y_value]
                self._compile_and_check(variable,
                                        [self.op(*variable)],
                                        data,
                                        self.op_class)

    def test_grad(self):
        for format in sparse.sparse_formats:
            for dtype in sparse.float_dtypes:
                (x, ), (x_value, ) = sparse_random_inputs(format,
                                                          shape=(9, 10),
                                                          out_dtype=dtype,
                                                          p=0.1)
                (y, ), (y_value, ) = sparse_random_inputs(format,
                                                          shape=(10, 24),
                                                          out_dtype=dtype,
                                                          p=0.1)
                variable = [x, y]
                data = [x_value, y_value]
                verify_grad_sparse(
                    self.op,
                    data,
                    structured=False)


class SamplingDotTester(utt.InferShapeTester):
    x = [tensor.matrix() for t in range(2)]
    x.append(sparse.csr_matrix())
    # unsquare shape
    a = [np.array(np.random.randint(1, 6, size=(4, 3)) - 1,
                     dtype=theano.config.floatX),
         np.array(np.random.randint(1, 6, size=(5, 3)) - 1,
                     dtype=theano.config.floatX),
         np.array(np.random.randint(1, 3, size=(4, 5)) - 1,
                     dtype=theano.config.floatX)
         ]
    a[2] = sp.csr_matrix(a[2])

    def setUp(self):
        super(SamplingDotTester, self).setUp()
        self.op_class = SamplingDot

    def test_op(self):
        f = theano.function(
            self.x,
            sampling_dot(*self.x))

        tested = f(*self.a)
        x, y, p = self.a
        expected = p.multiply(np.dot(x, y.T))

        utt.assert_allclose(as_ndarray(expected), tested.toarray())
        assert tested.format == 'csr'
        assert tested.dtype == expected.dtype

    def test_negative_stride(self):
        f = theano.function(
            self.x,
            sampling_dot(*self.x))

        a2 = [self.a[0][::-1,:], self.a[1][:,::-1], self.a[2]]
        tested = f(*a2)
        x, y, p = a2
        expected = p.multiply(np.dot(x, y.T))

        utt.assert_allclose(as_ndarray(expected), tested.toarray())
        assert tested.format == 'csr'
        assert tested.dtype == expected.dtype

    def test_infer_shape(self):
        self._compile_and_check(self.x,
                                [sampling_dot(*self.x)],
                                self.a,
                                self.op_class,
                                excluding=['local_sampling_dot_csr'])

    def test_grad(self):
        def _helper(x, y):
            return sampling_dot(x, y, self.a[2])
        verify_grad_sparse(_helper, self.a[:2])


import theano.tensor.tests.test_sharedvar
test_shared_options = theano.tensor.tests.test_sharedvar.makeSharedTester(
    shared_constructor_=theano.sparse.shared,
    dtype_='float64',
    get_value_borrow_true_alias_=True,
    shared_borrow_true_alias_=True,
    set_value_borrow_true_alias_=True,
    set_value_inplace_=False,
    set_cast_value_inplace_=False,
    shared_constructor_accept_ndarray_=False,
    internal_type_=scipy.sparse.csc_matrix,
    test_internal_type_=scipy.sparse.issparse,
    theano_fct_=lambda a: dense_from_sparse(a * 2.),
    ref_fct_=lambda a: np.asarray((a * 2).todense()),
    cast_value_=scipy.sparse.csr_matrix,
    name='test_shared_options',
)


if __name__ == '__main__':
    unittest.main()
