From d7b2159b33e6a537d4ae0e785e4098fd83900210 Mon Sep 17 00:00:00 2001 From: Maria Jose Molina Date: Wed, 30 Oct 2019 23:23:14 +0100 Subject: [PATCH] Format files with black --- ecopy/__init__.py | 2 +- ecopy/base_funcs/impute.py | 374 ++++++------- ecopy/base_funcs/spatial_median.py | 28 +- ecopy/base_funcs/wt_mean.py | 27 +- ecopy/base_funcs/wt_scale.py | 31 +- ecopy/base_funcs/wt_var.py | 39 +- ecopy/diversity/__init__.py | 2 +- ecopy/diversity/beta_dispersion.py | 336 ++++++------ ecopy/diversity/div_partition.py | 188 +++---- ecopy/diversity/diversity.py | 184 ++++--- ecopy/diversity/rarefy.py | 217 ++++---- ecopy/matrix_comp/anosim.py | 341 +++++++----- ecopy/matrix_comp/bioenv.py | 48 +- ecopy/matrix_comp/cca.py | 477 ++++++++++------- ecopy/matrix_comp/ccor.py | 252 +++++---- ecopy/matrix_comp/fourthcorner.py | 458 ++++++++-------- ecopy/matrix_comp/mantel.py | 321 ++++++------ ecopy/matrix_comp/procrust_test.py | 68 +-- ecopy/matrix_comp/rda.py | 474 ++++++++++------- ecopy/matrix_comp/rlq.py | 390 ++++++++------ ecopy/matrix_comp/simper.py | 151 +++--- ecopy/ordination/correspondance.py | 344 +++++++----- ecopy/ordination/distance.py | 814 +++++++++++++++-------------- ecopy/ordination/hillsmith.py | 320 +++++++----- ecopy/ordination/mds.py | 656 +++++++++++++---------- ecopy/ordination/ord_plot.py | 189 ++++--- ecopy/ordination/pca.py | 228 ++++---- ecopy/ordination/pcoa.py | 320 +++++++----- ecopy/ordination/transform.py | 247 +++++---- ecopy/regression/__init__.py | 2 +- ecopy/regression/isoregress.py | 106 ++-- ecopy/regression/nls.py | 152 +++--- ecopy/test_ecopy_unittest.py | 71 +-- ecopy/utils.py | 11 +- 34 files changed, 4355 insertions(+), 3513 deletions(-) diff --git a/ecopy/__init__.py b/ecopy/__init__.py index 85e8bdc..dce32d3 100644 --- a/ecopy/__init__.py +++ b/ecopy/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.1.2.2' +__version__ = "0.1.2.2" from .regression import * diff --git a/ecopy/base_funcs/impute.py b/ecopy/base_funcs/impute.py index e87eaf4..10a0c18 100644 --- a/ecopy/base_funcs/impute.py +++ b/ecopy/base_funcs/impute.py @@ -1,8 +1,9 @@ import numpy as np from pandas import DataFrame, Series -def impute(Y, method='mice', m=5, delta=0.0001, niter=100): - """ + +def impute(Y, method="mice", m=5, delta=0.0001, niter=100): + """ Docstring for function ecopy.impute ==================== Performs univariate missing data imputation using one of several methods described below @@ -55,204 +56,203 @@ def impute(Y, method='mice', m=5, delta=0.0001, niter=100): meanImpute = ep.impute(data, 'mean') """ - if not isinstance(Y, (Series, DataFrame, np.ndarray)): - msg = 'Y must be a pandas.Series, pandas.DataFrame, or numpy.ndarray' - raise ValueError(msg) - if delta < 0 or delta >= 0.1: - msg = 'k should be 0 <= k < 0.1' - raise ValueError(msg) - if method not in ['mean', 'median', 'multi_norm', 'univariate', 'monotone', 'mice']: - msg = 'method is not an approved imputation technique' - raise ValueError(msg) - fullData = np.array(Y) - completeObs = np.apply_along_axis(lambda x: np.sum(np.isnan(x)), 1, fullData) - obsData = fullData[completeObs==0,:] - if obsData.shape[0]==1: - msg = 'Only one row of complete observations' - raise ValueError(msg) - if method=='mean': - result = meanFunc(fullData, obsData) - if method=='median': - result = medianFunc(fullData, obsData) - if method=='multi_norm': - result = [] - k = 0 - while k < m: - tempR = multinormFunc(fullData, obsData) - result.append(tempR) - k += 1 - if method=='univariate': - result = [] - k = 0 - while k < m: - tempR = unipostFunc(fullData, delta) - result.append(tempR) - k += 1 - if method=='monotone': - result = [] - k = 0 - while k < m: - tempR = monotoneFunc(fullData, delta) - result.append(tempR) - k += 1 - if method=='mice': - result = [] - k = 0 - while k < m: - tempR = miceFunc(fullData, delta, niter) - result.append(tempR) - k += 1 - return result + if not isinstance(Y, (Series, DataFrame, np.ndarray)): + msg = "Y must be a pandas.Series, pandas.DataFrame, or numpy.ndarray" + raise ValueError(msg) + if delta < 0 or delta >= 0.1: + msg = "k should be 0 <= k < 0.1" + raise ValueError(msg) + if method not in ["mean", "median", "multi_norm", "univariate", "monotone", "mice"]: + msg = "method is not an approved imputation technique" + raise ValueError(msg) + fullData = np.array(Y) + completeObs = np.apply_along_axis(lambda x: np.sum(np.isnan(x)), 1, fullData) + obsData = fullData[completeObs == 0, :] + if obsData.shape[0] == 1: + msg = "Only one row of complete observations" + raise ValueError(msg) + if method == "mean": + result = meanFunc(fullData, obsData) + if method == "median": + result = medianFunc(fullData, obsData) + if method == "multi_norm": + result = [] + k = 0 + while k < m: + tempR = multinormFunc(fullData, obsData) + result.append(tempR) + k += 1 + if method == "univariate": + result = [] + k = 0 + while k < m: + tempR = unipostFunc(fullData, delta) + result.append(tempR) + k += 1 + if method == "monotone": + result = [] + k = 0 + while k < m: + tempR = monotoneFunc(fullData, delta) + result.append(tempR) + k += 1 + if method == "mice": + result = [] + k = 0 + while k < m: + tempR = miceFunc(fullData, delta, niter) + result.append(tempR) + k += 1 + return result def meanFunc(fullMat, obsMat): - z = fullMat.copy() - means = obsMat.mean(axis=0) - for i in range(len(means)): - idx = np.isnan(z[:,i]) - z[idx,i] = means[i] - return z + z = fullMat.copy() + means = obsMat.mean(axis=0) + for i in range(len(means)): + idx = np.isnan(z[:, i]) + z[idx, i] = means[i] + return z + def medianFunc(fullMat, obsMat): - z = fullMat.copy() - medians = np.median(obsMat, axis=0) - for i in range(len(medians)): - idx = np.isnan(z[:,i]) - z[idx,i] = medians[i] - return z + z = fullMat.copy() + medians = np.median(obsMat, axis=0) + for i in range(len(medians)): + idx = np.isnan(z[:, i]) + z[idx, i] = medians[i] + return z + def multinormFunc(fullMat, obsMat): - z = fullMat.copy() - means = obsMat.mean(axis=0) - cov = np.cov(obsMat, rowvar=0) - randoms = np.random.multivariate_normal(means, cov, fullMat.shape[0]*10) - for i in range(len(means)): - idx = np.isnan(z[:,i]) - randVals = np.random.choice(randoms[:,i], idx.sum(), replace=False) - z[idx,i] = randVals - return z + z = fullMat.copy() + means = obsMat.mean(axis=0) + cov = np.cov(obsMat, rowvar=0) + randoms = np.random.multivariate_normal(means, cov, fullMat.shape[0] * 10) + for i in range(len(means)): + idx = np.isnan(z[:, i]) + randVals = np.random.choice(randoms[:, i], idx.sum(), replace=False) + z[idx, i] = randVals + return z def unipostFunc(fullMat, delta): - z = fullMat.copy() - r = fullMat.copy() - for i in range(z.shape[1]): - yj = z[:,i] - xj = np.delete(z, i, axis=1) - idx = np.isnan(yj) - if idx.sum()==0: - r[:,i] = yj - else: - yj_obs = yj[~idx] - xj_obs = xj[~idx,:] - xj_nan = np.apply_along_axis(lambda x: np.sum(np.isnan(x)), 1, xj_obs) - xj_obs = xj_obs[xj_nan==0,:] - yj_obs = yj_obs[xj_nan==0] - S = xj_obs.T.dot(xj_obs) - V = np.linalg.pinv(S + np.diag(np.diag(S))*delta) - b_obs = V.dot(xj_obs.T).dot(yj_obs) - g = np.random.chisquare(xj_obs.shape[0] - xj_obs.shape[1]) - resid = yj_obs - xj_obs.dot(b_obs) - MSE = np.sqrt((resid.T.dot(resid))/g) - w1 = np.random.randn(xj_obs.shape[1]) - G = np.linalg.cholesky(V) - b_star = b_obs + MSE*w1.dot(V.T) - w2 = np.random.randn(idx.sum()) - xmis = xj[idx] - for k in range(xmis.shape[1]): - idx2 = np.isnan(xmis[:,k]) - xmis[idx2,k] = np.nanmean(xmis[:,k]) - y_star = xmis.dot(b_star) + w2*MSE - yj[idx] = y_star - r[:,i] = yj - return r + z = fullMat.copy() + r = fullMat.copy() + for i in range(z.shape[1]): + yj = z[:, i] + xj = np.delete(z, i, axis=1) + idx = np.isnan(yj) + if idx.sum() == 0: + r[:, i] = yj + else: + yj_obs = yj[~idx] + xj_obs = xj[~idx, :] + xj_nan = np.apply_along_axis(lambda x: np.sum(np.isnan(x)), 1, xj_obs) + xj_obs = xj_obs[xj_nan == 0, :] + yj_obs = yj_obs[xj_nan == 0] + S = xj_obs.T.dot(xj_obs) + V = np.linalg.pinv(S + np.diag(np.diag(S)) * delta) + b_obs = V.dot(xj_obs.T).dot(yj_obs) + g = np.random.chisquare(xj_obs.shape[0] - xj_obs.shape[1]) + resid = yj_obs - xj_obs.dot(b_obs) + MSE = np.sqrt((resid.T.dot(resid)) / g) + w1 = np.random.randn(xj_obs.shape[1]) + G = np.linalg.cholesky(V) + b_star = b_obs + MSE * w1.dot(V.T) + w2 = np.random.randn(idx.sum()) + xmis = xj[idx] + for k in range(xmis.shape[1]): + idx2 = np.isnan(xmis[:, k]) + xmis[idx2, k] = np.nanmean(xmis[:, k]) + y_star = xmis.dot(b_star) + w2 * MSE + yj[idx] = y_star + r[:, i] = yj + return r def monotoneFunc(fullMat, delta): - z = fullMat.copy() - r = fullMat.copy() - nMiss = np.apply_along_axis(lambda x: np.sum(np.isnan(x)), 0, z) - sortID = nMiss.argsort() - origID = np.arange(z.shape[1]) - orderedZ = z[:,sortID] - orderedR = r[:,sortID] - originalOrder = origID[sortID] - for i in range(orderedZ.shape[1]): - yj = orderedZ[:,i] - idx = np.isnan(yj) - if i==0: - yj[idx] = yj.mean() - orderedR[:,i] = yj - else: - xj = orderedR[:,range(i)] - yj_obs = yj[~idx] - xj_obs = xj[~idx,:] - S = xj_obs.T.dot(xj_obs) - V = np.linalg.pinv(S + np.diag(np.diag(S))*delta) - b_obs = V.dot(xj_obs.T).dot(yj_obs) - g = np.random.chisquare(xj_obs.shape[0] - xj_obs.shape[1]) - resid = yj_obs - xj_obs.dot(b_obs) - MSE = np.sqrt((resid.T.dot(resid))/g) - w1 = np.random.randn(xj_obs.shape[1]) - G = np.linalg.cholesky(V) - b_star = b_obs + MSE*w1.dot(V.T) - w2 = np.random.randn(idx.sum()) - xmis = xj[idx] - for k in range(xmis.shape[1]): - idx2 = np.isnan(xmis[:,k]) - xmis[idx2,k] = np.nanmean(xmis[:,k]) - y_star = xmis.dot(b_star) + w2*MSE - yj[idx] = y_star - orderedR[:,i] = yj - rFinal = orderedR[:,originalOrder] - return rFinal + z = fullMat.copy() + r = fullMat.copy() + nMiss = np.apply_along_axis(lambda x: np.sum(np.isnan(x)), 0, z) + sortID = nMiss.argsort() + origID = np.arange(z.shape[1]) + orderedZ = z[:, sortID] + orderedR = r[:, sortID] + originalOrder = origID[sortID] + for i in range(orderedZ.shape[1]): + yj = orderedZ[:, i] + idx = np.isnan(yj) + if i == 0: + yj[idx] = yj.mean() + orderedR[:, i] = yj + else: + xj = orderedR[:, range(i)] + yj_obs = yj[~idx] + xj_obs = xj[~idx, :] + S = xj_obs.T.dot(xj_obs) + V = np.linalg.pinv(S + np.diag(np.diag(S)) * delta) + b_obs = V.dot(xj_obs.T).dot(yj_obs) + g = np.random.chisquare(xj_obs.shape[0] - xj_obs.shape[1]) + resid = yj_obs - xj_obs.dot(b_obs) + MSE = np.sqrt((resid.T.dot(resid)) / g) + w1 = np.random.randn(xj_obs.shape[1]) + G = np.linalg.cholesky(V) + b_star = b_obs + MSE * w1.dot(V.T) + w2 = np.random.randn(idx.sum()) + xmis = xj[idx] + for k in range(xmis.shape[1]): + idx2 = np.isnan(xmis[:, k]) + xmis[idx2, k] = np.nanmean(xmis[:, k]) + y_star = xmis.dot(b_star) + w2 * MSE + yj[idx] = y_star + orderedR[:, i] = yj + rFinal = orderedR[:, originalOrder] + return rFinal def miceFunc(fullMat, delta, niter): - z = fullMat.copy() - r = fullMat.copy() - for i in range(z.shape[1]): - yj = z[:,i] - idx = np.isnan(yj) - if idx.sum()==0: - r[:,i] = yj - else: - yobs = yj[~idx] - yj[idx] = np.random.choice(yobs, idx.sum(), replace=False) - r[:,i] = yj - t = 0 - while t < niter: - for i in range(r.shape[1]): - yj = r[:,i] - xj = np.delete(r, i, axis=1) - idx = np.isnan(yj) - if idx.sum()==0: - r[:,i] = yj - else: - yj_obs = yj[~idx] - xj_obs = xj[~idx,:] - xj_nan = np.apply_along_axis(lambda x: np.sum(np.isnan(x)), 1, xj_obs) - xj_obs = xj_obs[xj_nan==0,:] - yj_obs = yj_obs[xj_nan==0] - S = xj_obs.T.dot(xj_obs) - V = np.linalg.pinv(S + np.diag(np.diag(S))*delta) - b_obs = V.dot(xj_obs.T).dot(yj_obs) - g = np.random.chisquare(xj_obs.shape[0] - xj_obs.shape[1]) - resid = yj_obs - xj_obs.dot(b_obs) - MSE = np.sqrt((resid.T.dot(resid))/g) - w1 = np.random.randn(xj_obs.shape[1]) - G = np.linalg.cholesky(V) - b_star = b_obs + MSE*w1.dot(V.T) - w2 = np.random.randn(idx.sum()) - xmis = xj[idx] - for k in range(xmis.shape[1]): - idx2 = np.isnan(xmis[:,k]) - xmis[idx2,k] = np.nanmean(xmis[:,k]) - y_star = xmis.dot(b_star) + w2*MSE - yj[idx] = y_star - r[:,i] = yj - t += 1 - return r - - - + z = fullMat.copy() + r = fullMat.copy() + for i in range(z.shape[1]): + yj = z[:, i] + idx = np.isnan(yj) + if idx.sum() == 0: + r[:, i] = yj + else: + yobs = yj[~idx] + yj[idx] = np.random.choice(yobs, idx.sum(), replace=False) + r[:, i] = yj + t = 0 + while t < niter: + for i in range(r.shape[1]): + yj = r[:, i] + xj = np.delete(r, i, axis=1) + idx = np.isnan(yj) + if idx.sum() == 0: + r[:, i] = yj + else: + yj_obs = yj[~idx] + xj_obs = xj[~idx, :] + xj_nan = np.apply_along_axis(lambda x: np.sum(np.isnan(x)), 1, xj_obs) + xj_obs = xj_obs[xj_nan == 0, :] + yj_obs = yj_obs[xj_nan == 0] + S = xj_obs.T.dot(xj_obs) + V = np.linalg.pinv(S + np.diag(np.diag(S)) * delta) + b_obs = V.dot(xj_obs.T).dot(yj_obs) + g = np.random.chisquare(xj_obs.shape[0] - xj_obs.shape[1]) + resid = yj_obs - xj_obs.dot(b_obs) + MSE = np.sqrt((resid.T.dot(resid)) / g) + w1 = np.random.randn(xj_obs.shape[1]) + G = np.linalg.cholesky(V) + b_star = b_obs + MSE * w1.dot(V.T) + w2 = np.random.randn(idx.sum()) + xmis = xj[idx] + for k in range(xmis.shape[1]): + idx2 = np.isnan(xmis[:, k]) + xmis[idx2, k] = np.nanmean(xmis[:, k]) + y_star = xmis.dot(b_star) + w2 * MSE + yj[idx] = y_star + r[:, i] = yj + t += 1 + return r diff --git a/ecopy/base_funcs/spatial_median.py b/ecopy/base_funcs/spatial_median.py index f18a597..35d86c8 100644 --- a/ecopy/base_funcs/spatial_median.py +++ b/ecopy/base_funcs/spatial_median.py @@ -2,8 +2,9 @@ from pandas import DataFrame from scipy.optimize import minimize + def spatial_median(X): - """ + """ Docstring for function ecopy.spatial_median ==================== Calculates the spatial median of a multivariate dataset. @@ -28,17 +29,18 @@ def spatial_median(X): data = multivariate_normal.rvs([0,0,0], cov, (100,)) spatialMed = ep.spatial_median(data) """ - if not isinstance(X, (DataFrame, np.ndarray)): - msg = 'X must be a pandas.DataFrame or numpy.ndarray' - raise ValueError(msg) - X = np.array(X) - medians = np.apply_along_axis(np.median, 0, X) - output = minimize(medianSearch, medians, args=(X,), method='BFGS') - finalMedians = output['x'] - return(finalMedians) + if not isinstance(X, (DataFrame, np.ndarray)): + msg = "X must be a pandas.DataFrame or numpy.ndarray" + raise ValueError(msg) + X = np.array(X) + medians = np.apply_along_axis(np.median, 0, X) + output = minimize(medianSearch, medians, args=(X,), method="BFGS") + finalMedians = output["x"] + return finalMedians + def medianSearch(params, Z): - med_iter = params - eucDist = np.apply_along_axis(lambda z: np.sqrt(np.sum((z-med_iter)**2)), 1, Z) - err = eucDist - return(np.mean(err)) \ No newline at end of file + med_iter = params + eucDist = np.apply_along_axis(lambda z: np.sqrt(np.sum((z - med_iter) ** 2)), 1, Z) + err = eucDist + return np.mean(err) diff --git a/ecopy/base_funcs/wt_mean.py b/ecopy/base_funcs/wt_mean.py index 94365eb..25e56c3 100644 --- a/ecopy/base_funcs/wt_mean.py +++ b/ecopy/base_funcs/wt_mean.py @@ -1,16 +1,17 @@ import numpy as np + def wt_mean(x, wt=None): - if wt is None: - wt = np.array([1]*len(x)) - x = np.array(x, 'float') - wt_array = np.array(wt, 'float') - if np.isnan(np.sum(x)): - msg = 'vector contains null values' - raise ValueError(msg) - if x.shape != wt_array.shape: - msg = 'weight vector must have equal dimensions as observations' - raise ValueError(msg) - wt_array = wt_array / wt_array.sum() - w_mu = np.sum(x*wt_array) - return w_mu \ No newline at end of file + if wt is None: + wt = np.array([1] * len(x)) + x = np.array(x, "float") + wt_array = np.array(wt, "float") + if np.isnan(np.sum(x)): + msg = "vector contains null values" + raise ValueError(msg) + if x.shape != wt_array.shape: + msg = "weight vector must have equal dimensions as observations" + raise ValueError(msg) + wt_array = wt_array / wt_array.sum() + w_mu = np.sum(x * wt_array) + return w_mu diff --git a/ecopy/base_funcs/wt_scale.py b/ecopy/base_funcs/wt_scale.py index c853ea5..cf7ce48 100644 --- a/ecopy/base_funcs/wt_scale.py +++ b/ecopy/base_funcs/wt_scale.py @@ -1,19 +1,20 @@ import numpy as np from ..base_funcs import wt_mean, wt_var + def wt_scale(x, wt, bias=0): - if wt is None: - wt = np.array([1]*len(x)) - x = np.array(x, 'float') - wt_array = np.array(wt, 'float') - if np.isnan(np.sum(x)): - msg = 'vector contains null values' - raise ValueError(msg) - if x.shape != wt_array.shape: - msg = 'weight vector must have equal dimensions as observations' - raise ValueError(msg) - wt_array = wt_array / wt_array.sum() - w_mu = wt_mean(x, wt_array) - w_var = wt_var(x, wt_array, bias) - w_scale = (x - w_mu) / np.sqrt(w_var) - return w_scale \ No newline at end of file + if wt is None: + wt = np.array([1] * len(x)) + x = np.array(x, "float") + wt_array = np.array(wt, "float") + if np.isnan(np.sum(x)): + msg = "vector contains null values" + raise ValueError(msg) + if x.shape != wt_array.shape: + msg = "weight vector must have equal dimensions as observations" + raise ValueError(msg) + wt_array = wt_array / wt_array.sum() + w_mu = wt_mean(x, wt_array) + w_var = wt_var(x, wt_array, bias) + w_scale = (x - w_mu) / np.sqrt(w_var) + return w_scale diff --git a/ecopy/base_funcs/wt_var.py b/ecopy/base_funcs/wt_var.py index b585663..8710725 100644 --- a/ecopy/base_funcs/wt_var.py +++ b/ecopy/base_funcs/wt_var.py @@ -1,23 +1,24 @@ import numpy as np from ..base_funcs import wt_mean + def wt_var(x, wt, bias=0): - if wt is None: - wt = np.array([1]*len(x)) - x = np.array(x, 'float') - wt_array = np.array(wt, 'float') - if np.isnan(np.sum(x)): - msg = 'vector contains null values' - raise ValueError(msg) - if x.shape != wt_array.shape: - msg = 'weight vector must have equal dimensions as observations' - raise ValueError(msg) - wt_array = wt_array / wt_array.sum() - w_mu = wt_mean(x, wt_array) - if bias==0: - p1 = np.sum(wt_array*(x-w_mu)**2) - p2 = 1/(1 - (wt_array**2).sum()) - w_var = p2*p1 - if bias==1: - w_var = np.sum(wt_array*(x - w_mu)**2) - return w_var + if wt is None: + wt = np.array([1] * len(x)) + x = np.array(x, "float") + wt_array = np.array(wt, "float") + if np.isnan(np.sum(x)): + msg = "vector contains null values" + raise ValueError(msg) + if x.shape != wt_array.shape: + msg = "weight vector must have equal dimensions as observations" + raise ValueError(msg) + wt_array = wt_array / wt_array.sum() + w_mu = wt_mean(x, wt_array) + if bias == 0: + p1 = np.sum(wt_array * (x - w_mu) ** 2) + p2 = 1 / (1 - (wt_array ** 2).sum()) + w_var = p2 * p1 + if bias == 1: + w_var = np.sum(wt_array * (x - w_mu) ** 2) + return w_var diff --git a/ecopy/diversity/__init__.py b/ecopy/diversity/__init__.py index 9e86336..0186c8c 100644 --- a/ecopy/diversity/__init__.py +++ b/ecopy/diversity/__init__.py @@ -1,4 +1,4 @@ from .diversity import diversity from .div_partition import div_partition from .rarefy import rarefy -from .beta_dispersion import beta_dispersion \ No newline at end of file +from .beta_dispersion import beta_dispersion diff --git a/ecopy/diversity/beta_dispersion.py b/ecopy/diversity/beta_dispersion.py index d7263d9..2e11815 100644 --- a/ecopy/diversity/beta_dispersion.py +++ b/ecopy/diversity/beta_dispersion.py @@ -3,8 +3,9 @@ from scipy.stats import f from ..base_funcs import spatial_median -def beta_dispersion(X, groups, test='anova', scores=False, center='median', n_iter=99): - ''' + +def beta_dispersion(X, groups, test="anova", scores=False, center="median", n_iter=99): + """ Docstring for function ecopy.beta_dispersion ======================== Calculates beta dispersion among groups for a given distance matrix. @@ -38,153 +39,190 @@ def beta_dispersion(X, groups, test='anova', scores=False, center='median', n_it groups = ['grazed']*16 + ['ungrazed']*8 print(ep.beta_dispersion(dist, groups, test='permute', center='median', scores=False)) - ''' - if not isinstance(X, (np.ndarray, DataFrame)): - msg = 'X must be a numpy.ndarray or pandas.DataFrame' - raise ValueError(msg) - if not isinstance(groups, (list, Series, DataFrame)): - msg = 'groups must be a list, pandas.Series, or pandas.DataFrame' - raise ValueError(msg) - Y = np.array(X) - groups = list(groups) - if np.isnan(Y).any(): - msg = 'Distance matrix contains null values' - raise ValueError(msg) - if Y.any() < 0: - msg ='Distance matrix cannot contain negative values' - raise ValueError(msg) - if Y.shape[0] != Y.shape[1]: - msg = 'Distance matrix must be square' - raise ValueError(msg) - if not np.allclose(Y.T, Y): - msg ='Distance matrix must be symmetric' - raise ValueError(msg) - if len(groups) != Y.shape[0]: - msg = 'Number of groups does not equal the rows of the distance matrix' - raise ValueError(msg) - if center not in ['median', 'centroid']: - msg = 'center must be either median or centroid' - raise ValueError(msg) - if test not in ['anova', 'permute']: - msg = 'test must be either anova or permute' - raise ValueError(msg) - A = -0.5*Y**2 - r_means = A.mean(axis=1) - c_means = A.mean(axis=0) - o_mean = A.mean() - G = A - r_means[:,np.newaxis] - c_means[np.newaxis,:] + o_mean - evals, U = np.linalg.eig(G) - idx = np.argsort(evals)[::-1] - evals = evals[idx] - U = U[:,idx] - Usc = U.dot(np.diag(np.abs(evals)**0.5)) - if test=='anova': - obs = z_calc(Usc, groups, evals, center, n_iter) - print('{0:<10} {1:^5} {2:^8} {3:^8} {4:^8} {5:^8}'.format(' ','df', 'SS', 'MS', 'F', 'P(>F)')) - print('{0:<10} {1:^5} {2:^8.4f} {3:^8.4f} {4:^8.4f} {5:^8.4f}'.format('Groups', int(obs[4]), obs[0], obs[2], obs[6], 1-f.cdf(obs[6], obs[4], obs[5]))) - print('{0:<10} {1:^5} {2:^8.4f} {3:^8.4f}'.format('Residuals', int(obs[5]), obs[1], obs[3])) - if test=='permute': - obs = z_calc(Usc, groups, evals, center, permute=True, iterations=n_iter) - print('{0:<10} {1:^5} {2:^8} {3:^8} {4:^8} {5:^8}'.format(' ','df', 'SS', 'MS', 'F', 'P(>F)')) - print('{0:<10} {1:^5} {2:^8.4f} {3:^8.4f} {4:^8.4f} {5:^8.4f}'.format('Groups', int(obs[4]), obs[0], obs[2], obs[6][0], np.mean(obs[6] > obs[6][0]))) - print('{0:<10} {1:^5} {2:^8.4f} {3:^8.4f}'.format('Residuals', int(obs[5]), obs[1], obs[3])) - if scores: - return obs[7] + """ + if not isinstance(X, (np.ndarray, DataFrame)): + msg = "X must be a numpy.ndarray or pandas.DataFrame" + raise ValueError(msg) + if not isinstance(groups, (list, Series, DataFrame)): + msg = "groups must be a list, pandas.Series, or pandas.DataFrame" + raise ValueError(msg) + Y = np.array(X) + groups = list(groups) + if np.isnan(Y).any(): + msg = "Distance matrix contains null values" + raise ValueError(msg) + if Y.any() < 0: + msg = "Distance matrix cannot contain negative values" + raise ValueError(msg) + if Y.shape[0] != Y.shape[1]: + msg = "Distance matrix must be square" + raise ValueError(msg) + if not np.allclose(Y.T, Y): + msg = "Distance matrix must be symmetric" + raise ValueError(msg) + if len(groups) != Y.shape[0]: + msg = "Number of groups does not equal the rows of the distance matrix" + raise ValueError(msg) + if center not in ["median", "centroid"]: + msg = "center must be either median or centroid" + raise ValueError(msg) + if test not in ["anova", "permute"]: + msg = "test must be either anova or permute" + raise ValueError(msg) + A = -0.5 * Y ** 2 + r_means = A.mean(axis=1) + c_means = A.mean(axis=0) + o_mean = A.mean() + G = A - r_means[:, np.newaxis] - c_means[np.newaxis, :] + o_mean + evals, U = np.linalg.eig(G) + idx = np.argsort(evals)[::-1] + evals = evals[idx] + U = U[:, idx] + Usc = U.dot(np.diag(np.abs(evals) ** 0.5)) + if test == "anova": + obs = z_calc(Usc, groups, evals, center, n_iter) + print( + "{0:<10} {1:^5} {2:^8} {3:^8} {4:^8} {5:^8}".format( + " ", "df", "SS", "MS", "F", "P(>F)" + ) + ) + print( + "{0:<10} {1:^5} {2:^8.4f} {3:^8.4f} {4:^8.4f} {5:^8.4f}".format( + "Groups", + int(obs[4]), + obs[0], + obs[2], + obs[6], + 1 - f.cdf(obs[6], obs[4], obs[5]), + ) + ) + print( + "{0:<10} {1:^5} {2:^8.4f} {3:^8.4f}".format( + "Residuals", int(obs[5]), obs[1], obs[3] + ) + ) + if test == "permute": + obs = z_calc(Usc, groups, evals, center, permute=True, iterations=n_iter) + print( + "{0:<10} {1:^5} {2:^8} {3:^8} {4:^8} {5:^8}".format( + " ", "df", "SS", "MS", "F", "P(>F)" + ) + ) + print( + "{0:<10} {1:^5} {2:^8.4f} {3:^8.4f} {4:^8.4f} {5:^8.4f}".format( + "Groups", + int(obs[4]), + obs[0], + obs[2], + obs[6][0], + np.mean(obs[6] > obs[6][0]), + ) + ) + print( + "{0:<10} {1:^5} {2:^8.4f} {3:^8.4f}".format( + "Residuals", int(obs[5]), obs[1], obs[3] + ) + ) + if scores: + return obs[7] def z_calc(U, groups, evals, center, iterations, permute=False): - groupIDs = list(set(groups)) - n_groups = len(groupIDs) - n = len(groups) - means_groups = np.empty(n_groups) - n_wg = np.empty(n_groups) - z_final = np.empty(n) - res = np.empty(U.shape) - pos = evals>=0 - for i in range(n_groups): - idx = np.array(groups)==groupIDs[i] - pos_Z = U[:,pos][idx,:] - if center=='median': - pos_cent = spatial_median(pos_Z) - if center=='centroid': - pos_cent = pos_Z.mean(axis=0) - pos_euclidean = np.apply_along_axis(lambda t: np.sum((t-pos_cent)**2), 1, pos_Z) - res[np.ix_(idx, pos)] = pos_Z - pos_cent - neg_euclidean = 0 - neg_res = 0 - neg_cent = 0 - if np.sum(~pos) > 0: - neg_Z = U[:,~pos][idx,:] - if center=='median': - neg_cent = spatial_median(neg_Z) - if center=='centroid': - neg_cent = neg_Z.mean(axis=0) - neg_euclidean = np.apply_along_axis(lambda t: np.sum((t-neg_cent)**2), 1, neg_Z) - neg_res = neg_Z - neg_cent[np.newaxis,:] - res[np.ix_(idx, ~pos)] = neg_Z - neg_cent - temp_r = np.sqrt(np.abs(pos_euclidean-neg_euclidean)) - means_groups[i] = np.mean(temp_r) - n_wg[i] = len(temp_r) - z_final[idx] = temp_r - if not permute: - ovMean = np.sum(n_wg*means_groups)/np.sum(n_wg) - SS_bg = np.sum(n_wg*(means_groups-ovMean)**2) - SS_tot = np.sum((z_final - ovMean)**2) - SS_wg = SS_tot - SS_bg - df_bg = n_groups - 1. - df_wg = float(n-n_groups) - MS_bg = SS_bg / (df_bg) - MS_wg = SS_wg / (df_wg) - F = MS_bg / MS_wg - return SS_bg, SS_wg, MS_bg, MS_wg, df_bg, df_wg, F, z_final - if permute: - ovMean = np.sum(n_wg*means_groups)/np.sum(n_wg) - SS_bg = np.sum(n_wg*(means_groups-ovMean)**2) - SS_tot = np.sum((z_final - ovMean)**2) - SS_wg = SS_tot - SS_bg - df_bg = n_groups - 1. - df_wg = float(n-n_groups) - MS_bg = SS_bg / (df_bg) - MS_wg = SS_wg / (df_wg) - F = np.empty(iterations) - F[0] = MS_bg / MS_wg - k = 1 - while k < iterations: - index1 = np.arange(res.shape[0]) - res_perm_pos = res[np.ix_(np.random.choice(index1, len(index1), replace=False),pos)] - res_perm_pos = res_perm_pos + pos_cent[np.newaxis,:] - res_perm_neg = res[np.ix_(np.random.choice(index1, len(index1), replace=False),~pos)] - res_perm_neg = res_perm_neg + neg_cent[np.newaxis,:] - U_perm = np.concatenate((res_perm_pos, res_perm_neg), axis=1) - for i in range(n_groups): - idx = np.array(groups)==groupIDs[i] - pos_Z = U_perm[np.ix_(idx, pos)] - pos_euclidean = np.apply_along_axis(lambda t: np.sum((t-pos_cent)**2), 1, pos_Z) - neg_euclidean = 0 - if np.sum(~pos) > 0: - neg_Z = U_perm[np.ix_(idx,~pos)] - neg_euclidean = np.apply_along_axis(lambda t: np.sum((t-neg_cent)**2), 1, neg_Z) - temp_r = np.sqrt(np.abs(pos_euclidean-neg_euclidean)) - means_groups[i] = np.mean(temp_r) - n_wg[i] = len(temp_r) - z_final[idx] = temp_r - ovMean_perm = np.sum(n_wg*means_groups)/np.sum(n_wg) - SS_bg_perm = np.sum(n_wg*(means_groups-ovMean_perm)**2) - SS_tot_perm = np.sum((z_final - ovMean_perm)**2) - SS_wg_perm = SS_tot_perm - SS_bg_perm - df_bg = n_groups - 1. - df_wg = float(n-n_groups) - MS_bg_perm = SS_bg_perm / (df_bg) - MS_wg_perm = SS_wg_perm / (df_wg) - F[k] = MS_bg_perm / MS_wg_perm - k+=1 - return SS_bg, SS_wg, MS_bg, MS_wg, df_bg, df_wg, F, z_final - - - - - - - - - + groupIDs = list(set(groups)) + n_groups = len(groupIDs) + n = len(groups) + means_groups = np.empty(n_groups) + n_wg = np.empty(n_groups) + z_final = np.empty(n) + res = np.empty(U.shape) + pos = evals >= 0 + for i in range(n_groups): + idx = np.array(groups) == groupIDs[i] + pos_Z = U[:, pos][idx, :] + if center == "median": + pos_cent = spatial_median(pos_Z) + if center == "centroid": + pos_cent = pos_Z.mean(axis=0) + pos_euclidean = np.apply_along_axis( + lambda t: np.sum((t - pos_cent) ** 2), 1, pos_Z + ) + res[np.ix_(idx, pos)] = pos_Z - pos_cent + neg_euclidean = 0 + neg_res = 0 + neg_cent = 0 + if np.sum(~pos) > 0: + neg_Z = U[:, ~pos][idx, :] + if center == "median": + neg_cent = spatial_median(neg_Z) + if center == "centroid": + neg_cent = neg_Z.mean(axis=0) + neg_euclidean = np.apply_along_axis( + lambda t: np.sum((t - neg_cent) ** 2), 1, neg_Z + ) + neg_res = neg_Z - neg_cent[np.newaxis, :] + res[np.ix_(idx, ~pos)] = neg_Z - neg_cent + temp_r = np.sqrt(np.abs(pos_euclidean - neg_euclidean)) + means_groups[i] = np.mean(temp_r) + n_wg[i] = len(temp_r) + z_final[idx] = temp_r + if not permute: + ovMean = np.sum(n_wg * means_groups) / np.sum(n_wg) + SS_bg = np.sum(n_wg * (means_groups - ovMean) ** 2) + SS_tot = np.sum((z_final - ovMean) ** 2) + SS_wg = SS_tot - SS_bg + df_bg = n_groups - 1.0 + df_wg = float(n - n_groups) + MS_bg = SS_bg / (df_bg) + MS_wg = SS_wg / (df_wg) + F = MS_bg / MS_wg + return SS_bg, SS_wg, MS_bg, MS_wg, df_bg, df_wg, F, z_final + if permute: + ovMean = np.sum(n_wg * means_groups) / np.sum(n_wg) + SS_bg = np.sum(n_wg * (means_groups - ovMean) ** 2) + SS_tot = np.sum((z_final - ovMean) ** 2) + SS_wg = SS_tot - SS_bg + df_bg = n_groups - 1.0 + df_wg = float(n - n_groups) + MS_bg = SS_bg / (df_bg) + MS_wg = SS_wg / (df_wg) + F = np.empty(iterations) + F[0] = MS_bg / MS_wg + k = 1 + while k < iterations: + index1 = np.arange(res.shape[0]) + res_perm_pos = res[ + np.ix_(np.random.choice(index1, len(index1), replace=False), pos) + ] + res_perm_pos = res_perm_pos + pos_cent[np.newaxis, :] + res_perm_neg = res[ + np.ix_(np.random.choice(index1, len(index1), replace=False), ~pos) + ] + res_perm_neg = res_perm_neg + neg_cent[np.newaxis, :] + U_perm = np.concatenate((res_perm_pos, res_perm_neg), axis=1) + for i in range(n_groups): + idx = np.array(groups) == groupIDs[i] + pos_Z = U_perm[np.ix_(idx, pos)] + pos_euclidean = np.apply_along_axis( + lambda t: np.sum((t - pos_cent) ** 2), 1, pos_Z + ) + neg_euclidean = 0 + if np.sum(~pos) > 0: + neg_Z = U_perm[np.ix_(idx, ~pos)] + neg_euclidean = np.apply_along_axis( + lambda t: np.sum((t - neg_cent) ** 2), 1, neg_Z + ) + temp_r = np.sqrt(np.abs(pos_euclidean - neg_euclidean)) + means_groups[i] = np.mean(temp_r) + n_wg[i] = len(temp_r) + z_final[idx] = temp_r + ovMean_perm = np.sum(n_wg * means_groups) / np.sum(n_wg) + SS_bg_perm = np.sum(n_wg * (means_groups - ovMean_perm) ** 2) + SS_tot_perm = np.sum((z_final - ovMean_perm) ** 2) + SS_wg_perm = SS_tot_perm - SS_bg_perm + df_bg = n_groups - 1.0 + df_wg = float(n - n_groups) + MS_bg_perm = SS_bg_perm / (df_bg) + MS_wg_perm = SS_wg_perm / (df_wg) + F[k] = MS_bg_perm / MS_wg_perm + k += 1 + return SS_bg, SS_wg, MS_bg, MS_wg, df_bg, df_wg, F, z_final diff --git a/ecopy/diversity/div_partition.py b/ecopy/diversity/div_partition.py index e673d20..bee0055 100644 --- a/ecopy/diversity/div_partition.py +++ b/ecopy/diversity/div_partition.py @@ -1,8 +1,9 @@ import numpy as np from pandas import DataFrame -def div_partition(x, method='shannon', breakNA=True, weights=None): - ''' + +def div_partition(x, method="shannon", breakNA=True, weights=None): + """ Docstring for function ecopy.diversity ======================== Decomposes diversity measures into alpha, beta, and gamma components. @@ -35,97 +36,108 @@ def div_partition(x, method='shannon', breakNA=True, weights=None): varespec = ep.load_data('varespec') D_alpha, D_beta, D_gamma = ep.div_partition(varespec, 'shannon') - ''' - listofmethods = ['shannon', 'gini-simpson', 'simpson', 'dominance', 'spRich', 'even'] - if not isinstance(breakNA, bool): - msg = 'removaNA argument must be boolean' - raise ValueError(msg) - if method not in listofmethods: - msg = 'method argument {0!s} is not an accepted metric'.format(method) - raise ValueError(msg) - if not isinstance(x, (DataFrame, np.ndarray)): - msg = 'x argument must be a numpy array or pandas dataframe' - raise ValueError(msg) - if isinstance(x, DataFrame): - if (x.dtypes == 'object').any(): - msg = 'DataFrame can only contain numeric values' - if breakNA: - if x.isnull().any().any(): - msg = 'DataFrame contains null values' - raise ValueError(msg) - if (x<0).any().any(): - msg = 'DataFrame contains negative values' - raise ValueError(msg) - z = np.array(x, 'float') - if isinstance(x, np.ndarray): - if breakNA: - if np.isnan(np.sum(x)): - msg = 'Array contains null values' - raise ValueError(msg) - if (x < 0).any(): - msg = 'Array contains negative values' - raise ValueError(msg) - z = np.array(x, 'float') - if weights is None: - weights = z.sum(axis=1)/z.sum() - w = np.array(weights)/np.array(weights).sum() - relMat = z / z.sum(axis=1)[:,np.newaxis] - totalList = z.sum(axis=0) - totalRel = totalList / totalList.sum() - if method=='shannon': - div = np.apply_along_axis(shannonFunc, 1, relMat) - H_alpha = (w*div).sum() / w.sum() - H_gamma = shannonFunc(totalRel) - D_alpha = np.exp(H_alpha) - D_gamma = np.exp(H_gamma) - D_beta = D_gamma / D_alpha - return D_alpha, D_beta, D_gamma - if method=='gini-simpson': - div = np.apply_along_axis(giniFunc, 1, relMat) - H_alpha = (w*div).sum()/w.sum() - H_gamma = giniFunc(totalRel) - D_alpha = 1./(1.-H_alpha) - D_gamma = 1./(1.-H_gamma) - D_beta = D_gamma / D_alpha - return D_alpha, D_beta, D_gamma - if method=='simpson': - div = np.apply_along_axis(simpson, 1, relMat) - H_alpha = (w*div).sum()/w.sum() - H_gamma = simpson(totalRel) - D_alpha = 1./H_alpha - D_gamma = 1./H_gamma - D_beta = D_gamma / D_alpha - return D_alpha, D_beta, D_gamma - if method=='spRich': - div = np.apply_along_axis(richness, 1, relMat) - H_alpha = (w*div).sum()/w.sum() - H_gamma = richness(totalRel) - D_alpha = H_alpha - D_gamma = H_gamma - D_beta = D_gamma / D_alpha - return D_alpha, D_beta, D_gamma + """ + listofmethods = [ + "shannon", + "gini-simpson", + "simpson", + "dominance", + "spRich", + "even", + ] + if not isinstance(breakNA, bool): + msg = "removaNA argument must be boolean" + raise ValueError(msg) + if method not in listofmethods: + msg = "method argument {0!s} is not an accepted metric".format(method) + raise ValueError(msg) + if not isinstance(x, (DataFrame, np.ndarray)): + msg = "x argument must be a numpy array or pandas dataframe" + raise ValueError(msg) + if isinstance(x, DataFrame): + if (x.dtypes == "object").any(): + msg = "DataFrame can only contain numeric values" + if breakNA: + if x.isnull().any().any(): + msg = "DataFrame contains null values" + raise ValueError(msg) + if (x < 0).any().any(): + msg = "DataFrame contains negative values" + raise ValueError(msg) + z = np.array(x, "float") + if isinstance(x, np.ndarray): + if breakNA: + if np.isnan(np.sum(x)): + msg = "Array contains null values" + raise ValueError(msg) + if (x < 0).any(): + msg = "Array contains negative values" + raise ValueError(msg) + z = np.array(x, "float") + if weights is None: + weights = z.sum(axis=1) / z.sum() + w = np.array(weights) / np.array(weights).sum() + relMat = z / z.sum(axis=1)[:, np.newaxis] + totalList = z.sum(axis=0) + totalRel = totalList / totalList.sum() + if method == "shannon": + div = np.apply_along_axis(shannonFunc, 1, relMat) + H_alpha = (w * div).sum() / w.sum() + H_gamma = shannonFunc(totalRel) + D_alpha = np.exp(H_alpha) + D_gamma = np.exp(H_gamma) + D_beta = D_gamma / D_alpha + return D_alpha, D_beta, D_gamma + if method == "gini-simpson": + div = np.apply_along_axis(giniFunc, 1, relMat) + H_alpha = (w * div).sum() / w.sum() + H_gamma = giniFunc(totalRel) + D_alpha = 1.0 / (1.0 - H_alpha) + D_gamma = 1.0 / (1.0 - H_gamma) + D_beta = D_gamma / D_alpha + return D_alpha, D_beta, D_gamma + if method == "simpson": + div = np.apply_along_axis(simpson, 1, relMat) + H_alpha = (w * div).sum() / w.sum() + H_gamma = simpson(totalRel) + D_alpha = 1.0 / H_alpha + D_gamma = 1.0 / H_gamma + D_beta = D_gamma / D_alpha + return D_alpha, D_beta, D_gamma + if method == "spRich": + div = np.apply_along_axis(richness, 1, relMat) + H_alpha = (w * div).sum() / w.sum() + H_gamma = richness(totalRel) + D_alpha = H_alpha + D_gamma = H_gamma + D_beta = D_gamma / D_alpha + return D_alpha, D_beta, D_gamma + def shannonFunc(y): - notabs = ~np.isnan(y) - t = y[notabs] / np.sum(y[notabs]) - t = t[t!=0] - H = -np.sum( t*np.log(t) ) - return H + notabs = ~np.isnan(y) + t = y[notabs] / np.sum(y[notabs]) + t = t[t != 0] + H = -np.sum(t * np.log(t)) + return H + def giniFunc(y): - notabs = ~np.isnan(y) - t = y[notabs] / np.sum(y[notabs]) - D = 1 - np.sum( t**2 ) - return D + notabs = ~np.isnan(y) + t = y[notabs] / np.sum(y[notabs]) + D = 1 - np.sum(t ** 2) + return D + def simpson(y): - notabs = ~np.isnan(y) - t = y[notabs] / np.sum(y[notabs]) - D = np.sum( t**2 ) - return D + notabs = ~np.isnan(y) + t = y[notabs] / np.sum(y[notabs]) + D = np.sum(t ** 2) + return D + def richness(y): - notabs = ~np.isnan(y) - t = y[notabs] - D = np.sum(t!=0) - return float(D) + notabs = ~np.isnan(y) + t = y[notabs] + D = np.sum(t != 0) + return float(D) diff --git a/ecopy/diversity/diversity.py b/ecopy/diversity/diversity.py index dd0679c..b082568 100644 --- a/ecopy/diversity/diversity.py +++ b/ecopy/diversity/diversity.py @@ -1,8 +1,9 @@ import numpy as np from pandas import DataFrame, Series -def diversity(x, method='shannon', breakNA=True, num_equiv=True): - ''' + +def diversity(x, method="shannon", breakNA=True, num_equiv=True): + """ Docstring for function ecopy.diversity ======================== Computes a given diversity index for a site x species matrix. @@ -50,98 +51,111 @@ def diversity(x, method='shannon', breakNA=True, num_equiv=True): varespec = ep.load_data('varespec') div = ep.diversity(varespec, 'shannon') - ''' - listofmethods = ['shannon', 'gini-simpson', 'simpson', 'dominance', 'spRich', 'even'] - if not isinstance(breakNA, bool): - msg = 'removaNA argument must be boolean' - raise ValueError(msg) - if method not in listofmethods: - msg = 'method argument {0!s} is not an accepted metric'.format(method) - raise ValueError(msg) - if not isinstance(x, (DataFrame, np.ndarray)): - msg = 'x argument must be a numpy array or pandas dataframe' - raise ValueError(msg) - if isinstance(x, DataFrame): - if (x.dtypes == 'object').any(): - msg = 'DataFrame can only contain numeric values' - if breakNA: - if x.isnull().any().any(): - msg = 'DataFrame contains null values' - raise ValueError(msg) - if (x<0).any().any(): - msg = 'DataFrame contains negative values' - raise ValueError(msg) - z = np.array(x, 'float') - if isinstance(x, np.ndarray): - if breakNA: - if np.isnan(np.sum(x)): - msg = 'Array contains null values' - raise ValueError(msg) - if (x < 0).any(): - msg = 'Array contains negative values' - raise ValueError(msg) - z = np.array(x, 'float') - z = z / z.sum(axis=1)[:,np.newaxis] - if method=='shannon': - div = np.apply_along_axis(shannonFunc, 1, z) - if num_equiv: - div = np.exp(div) - return div - if method=='gini-simpson': - div = np.apply_along_axis(giniFunc, 1, z) - if num_equiv: - div = 1./(1.-div) - return div - if method=='simpson': - div = np.apply_along_axis(simpson, 1, z) - if num_equiv: - div = 1./div - return div - if method=='dominance': - div = np.apply_along_axis(dom, 1, z) - return div - if method=='spRich': - div = np.apply_along_axis(richness, 1, z) - return div - if method=='even': - div = np.apply_along_axis(evenFunc, 1, z) - return div + """ + listofmethods = [ + "shannon", + "gini-simpson", + "simpson", + "dominance", + "spRich", + "even", + ] + if not isinstance(breakNA, bool): + msg = "removaNA argument must be boolean" + raise ValueError(msg) + if method not in listofmethods: + msg = "method argument {0!s} is not an accepted metric".format(method) + raise ValueError(msg) + if not isinstance(x, (DataFrame, np.ndarray)): + msg = "x argument must be a numpy array or pandas dataframe" + raise ValueError(msg) + if isinstance(x, DataFrame): + if (x.dtypes == "object").any(): + msg = "DataFrame can only contain numeric values" + if breakNA: + if x.isnull().any().any(): + msg = "DataFrame contains null values" + raise ValueError(msg) + if (x < 0).any().any(): + msg = "DataFrame contains negative values" + raise ValueError(msg) + z = np.array(x, "float") + if isinstance(x, np.ndarray): + if breakNA: + if np.isnan(np.sum(x)): + msg = "Array contains null values" + raise ValueError(msg) + if (x < 0).any(): + msg = "Array contains negative values" + raise ValueError(msg) + z = np.array(x, "float") + z = z / z.sum(axis=1)[:, np.newaxis] + if method == "shannon": + div = np.apply_along_axis(shannonFunc, 1, z) + if num_equiv: + div = np.exp(div) + return div + if method == "gini-simpson": + div = np.apply_along_axis(giniFunc, 1, z) + if num_equiv: + div = 1.0 / (1.0 - div) + return div + if method == "simpson": + div = np.apply_along_axis(simpson, 1, z) + if num_equiv: + div = 1.0 / div + return div + if method == "dominance": + div = np.apply_along_axis(dom, 1, z) + return div + if method == "spRich": + div = np.apply_along_axis(richness, 1, z) + return div + if method == "even": + div = np.apply_along_axis(evenFunc, 1, z) + return div + def shannonFunc(y): - notabs = ~np.isnan(y) - t = y[notabs] / np.sum(y[notabs]) - t = t[t!=0] - H = -np.sum( t*np.log(t) ) - return H + notabs = ~np.isnan(y) + t = y[notabs] / np.sum(y[notabs]) + t = t[t != 0] + H = -np.sum(t * np.log(t)) + return H + def giniFunc(y): - notabs = ~np.isnan(y) - t = y[notabs] / np.sum(y[notabs]) - D = 1 - np.sum( t**2 ) - return D + notabs = ~np.isnan(y) + t = y[notabs] / np.sum(y[notabs]) + D = 1 - np.sum(t ** 2) + return D + def simpson(y): - notabs = ~np.isnan(y) - t = y[notabs] / np.sum(y[notabs]) - D = np.sum( t**2 ) - return D + notabs = ~np.isnan(y) + t = y[notabs] / np.sum(y[notabs]) + D = np.sum(t ** 2) + return D + def dom(y): - notabs = ~np.isnan(y) - t = y[notabs] / np.sum(y[notabs]) - D = np.max(t) - return D + notabs = ~np.isnan(y) + t = y[notabs] / np.sum(y[notabs]) + D = np.max(t) + return D + def richness(y): - notabs = ~np.isnan(y) - t = y[notabs] - D = np.sum(t!=0) - return float(D) + notabs = ~np.isnan(y) + t = y[notabs] + D = np.sum(t != 0) + return float(D) + def evenFunc(y): - notabs = ~np.isnan(y) - t = y[notabs] / np.sum(y[notabs]) - n = float(np.sum(t!=0)) - t = t[t!=0] - H = -np.sum( t*np.log(t) ) - return H/np.log(n) + notabs = ~np.isnan(y) + t = y[notabs] / np.sum(y[notabs]) + n = float(np.sum(t != 0)) + t = t[t != 0] + H = -np.sum(t * np.log(t)) + return H / np.log(n) diff --git a/ecopy/diversity/rarefy.py b/ecopy/diversity/rarefy.py index 82d264e..9e04b64 100644 --- a/ecopy/diversity/rarefy.py +++ b/ecopy/diversity/rarefy.py @@ -3,8 +3,9 @@ from scipy.misc import comb import matplotlib.pyplot as plt -def rarefy(x, method='rarefy', size = None, breakNA=True): - ''' + +def rarefy(x, method="rarefy", size=None, breakNA=True): + """ Docstring for function ecopy.rarefy ======================== Various rarefaction techniques for a site x species matrix. @@ -46,114 +47,120 @@ def rarefy(x, method='rarefy', size = None, breakNA=True): # draw rarefaction curves ep.rarefy(BCI, 'rarecurve') - ''' - listofmethods = ['rarefy', 'rarecurve'] - if not isinstance(breakNA, bool): - msg = 'removaNA argument must be boolean' - raise ValueError(msg) - if method not in listofmethods: - msg = 'method argument {0!s} is not an accepted rarefaction method'.format(method) - raise ValueError(msg) - if not isinstance(x, (DataFrame, np.ndarray)): - msg = 'x argument must be a numpy array or pandas dataframe' - raise ValueError(msg) - if size is not None: - if not isinstance(size, (int, float, np.ndarray)): - msg = 'size must be integer, float, or numpy array' - raise ValueError(msg) - if isinstance(x, DataFrame): - if (x.dtypes == 'object').any(): - msg = 'DataFrame can only contain numeric values' - if breakNA: - if x.isnull().any().any(): - msg = 'DataFrame contains null values' - raise ValueError(msg) - if (x<0).any().any(): - msg = 'DataFrame contains negative values' - raise ValueError(msg) - if method=='rarefy': - if size is None: - sums = x.apply(sum, axis=1) - size = np.min(sums) - rich = x.apply(rare, axis=1, args=(size,)) - return rich - else: - if isinstance(size, (int, float)): - rich = x.apply(rare, axis=1, args=(size,)) - return rich - else: - if len(size) != len(x): - msg = 'length of size does not match number of rows' - raise ValueError(msg) - z = x.copy() - z['size'] = size - rich = z.apply(rare_wrapper, axis=1) - return rich - if method=='rarecurve': - z = x.copy() - z.reset_index(inplace=True) - z.apply(rCurve, axis=1) - plt.xlabel('Number of Individuals') - plt.ylabel('Number of Species') - plt.show() - if isinstance(x, np.ndarray): - if breakNA: - if np.isnan(np.sum(x)): - msg = 'Array contains null values' - raise ValueError(msg) - if (x < 0).any(): - msg = 'Array contains negative values' - raise ValueError(msg) - if method=='rarefy': - if size is None: - sums = np.apply_along_axis(np.nansum, 1, x) - size = np.min(sums) - rich = np.apply_along_axis(rare, 1, x, size) - return rich - else: - if isinstance(size, (int, float)): - rich = np.apply_along_axis(rare, 1, x, size) - return rich - else: - if len(size) != x.shape[0]: - msg = 'length of size does not match number of rows' - raise ValueError(msg) - N = np.nansum(x, axis=1) - diff = (N[:,np.newaxis] - x).T - return np.sum(1 - comb(diff, size)/comb(N, size), axis=0) - if method=='rarecurve': - z = DataFrame(x) - z.reset_index(inplace=True) - z.apply(rCurve, axis=1) - plt.xlabel('Number of Individuals') - plt.ylabel('Number of Species') - plt.show() + """ + listofmethods = ["rarefy", "rarecurve"] + if not isinstance(breakNA, bool): + msg = "removaNA argument must be boolean" + raise ValueError(msg) + if method not in listofmethods: + msg = "method argument {0!s} is not an accepted rarefaction method".format( + method + ) + raise ValueError(msg) + if not isinstance(x, (DataFrame, np.ndarray)): + msg = "x argument must be a numpy array or pandas dataframe" + raise ValueError(msg) + if size is not None: + if not isinstance(size, (int, float, np.ndarray)): + msg = "size must be integer, float, or numpy array" + raise ValueError(msg) + if isinstance(x, DataFrame): + if (x.dtypes == "object").any(): + msg = "DataFrame can only contain numeric values" + if breakNA: + if x.isnull().any().any(): + msg = "DataFrame contains null values" + raise ValueError(msg) + if (x < 0).any().any(): + msg = "DataFrame contains negative values" + raise ValueError(msg) + if method == "rarefy": + if size is None: + sums = x.apply(sum, axis=1) + size = np.min(sums) + rich = x.apply(rare, axis=1, args=(size,)) + return rich + else: + if isinstance(size, (int, float)): + rich = x.apply(rare, axis=1, args=(size,)) + return rich + else: + if len(size) != len(x): + msg = "length of size does not match number of rows" + raise ValueError(msg) + z = x.copy() + z["size"] = size + rich = z.apply(rare_wrapper, axis=1) + return rich + if method == "rarecurve": + z = x.copy() + z.reset_index(inplace=True) + z.apply(rCurve, axis=1) + plt.xlabel("Number of Individuals") + plt.ylabel("Number of Species") + plt.show() + if isinstance(x, np.ndarray): + if breakNA: + if np.isnan(np.sum(x)): + msg = "Array contains null values" + raise ValueError(msg) + if (x < 0).any(): + msg = "Array contains negative values" + raise ValueError(msg) + if method == "rarefy": + if size is None: + sums = np.apply_along_axis(np.nansum, 1, x) + size = np.min(sums) + rich = np.apply_along_axis(rare, 1, x, size) + return rich + else: + if isinstance(size, (int, float)): + rich = np.apply_along_axis(rare, 1, x, size) + return rich + else: + if len(size) != x.shape[0]: + msg = "length of size does not match number of rows" + raise ValueError(msg) + N = np.nansum(x, axis=1) + diff = (N[:, np.newaxis] - x).T + return np.sum(1 - comb(diff, size) / comb(N, size), axis=0) + if method == "rarecurve": + z = DataFrame(x) + z.reset_index(inplace=True) + z.apply(rCurve, axis=1) + plt.xlabel("Number of Individuals") + plt.ylabel("Number of Species") + plt.show() + def rare(y, size): - notabs = ~np.isnan(y) - t = y[notabs] - N = np.sum(t) - diff = N - t - rare_calc = np.sum(1 - comb(diff, size)/comb(N, size)) - return rare_calc + notabs = ~np.isnan(y) + t = y[notabs] + N = np.sum(t) + diff = N - t + rare_calc = np.sum(1 - comb(diff, size) / comb(N, size)) + return rare_calc + def rare_wrapper(data): - s2 = data['size'] - x2 = data.drop('size') - return rare(x2, s2) + s2 = data["size"] + x2 = data.drop("size") + return rare(x2, s2) + def rareCurve_Func(i, Sn, n, x): - sBar = Sn - np.sum(comb(n-x, i))/comb(n, i) - return sBar + sBar = Sn - np.sum(comb(n - x, i)) / comb(n, i) + return sBar + def rCurve(x): - ix = x['index'] - z = x.drop('index').astype('float') - notabs = ~np.isnan(z) - y = z[notabs] - n = np.sum(y) - Sn = len(z) - iPred = np.linspace(0, n, 1000) - yhat = [rareCurve_Func(i, Sn, n, y) for i in iPred] - plt.plot(iPred, yhat) - plt.text(iPred[-1], yhat[-1], str(ix), ha='left', va='center') + ix = x["index"] + z = x.drop("index").astype("float") + notabs = ~np.isnan(z) + y = z[notabs] + n = np.sum(y) + Sn = len(z) + iPred = np.linspace(0, n, 1000) + yhat = [rareCurve_Func(i, Sn, n, y) for i in iPred] + plt.plot(iPred, yhat) + plt.text(iPred[-1], yhat[-1], str(ix), ha="left", va="center") diff --git a/ecopy/matrix_comp/anosim.py b/ecopy/matrix_comp/anosim.py index 822b86f..2be193c 100644 --- a/ecopy/matrix_comp/anosim.py +++ b/ecopy/matrix_comp/anosim.py @@ -2,8 +2,9 @@ from pandas import DataFrame import matplotlib.pyplot as plt + class anosim(object): - ''' + """ Docstring for function ecopy.anosim ==================== Conducts analysis of similarity (ANOSIM) on a distance matrix given @@ -52,142 +53,212 @@ class anosim(object): t1 = ep.anosim(duneDist, group1, group2, nested=True, nperm=9999) print(t1.summary()) t1.plot() - ''' - def __init__(self, dist, factor1, factor2=None, nested=False, nperm=999): - if isinstance(dist, DataFrame): - dist = np.array(dist) - if dist.shape[0] != dist.shape[1]: - msg = 'Matrix dist must be a square, symmetric distance matrix' - raise ValueError(msg) - if not np.allclose(dist.T, dist): - msg = 'Matrix dist must be a square, symmetric distance matrix' - raise ValueError(msg) - if np.any(dist < 0): - msg = 'Distance matrix cannot have negative values' - raise ValueError(msg) - self.r_perm1 = np.empty(nperm) - self.r_perm2 = np.empty(nperm) - self.R_obs1 = None - self.R_obs2 = None - if factor2 is None: - g1 = np.array(factor1) - self.R_obs1 = oneWayANOSIM(dist, g1) - for i in range(nperm): - groupRand = np.random.choice(g1, len(g1), replace=False) - self.r_perm1[i] = oneWayANOSIM(dist, groupRand) - self.p_val = np.mean(self.r_perm1 > self.R_obs1) - if factor2 is not None and not nested: - g1 = np.array(factor1) - self.R_obs1 = oneWayANOSIM(dist, g1) - for i in range(nperm): - groupRand = np.random.choice(g1, len(g1), replace=False) - self.r_perm1[i] = oneWayANOSIM(dist, groupRand) - g2 = np.array(factor2) - self.R_obs2 = oneWayANOSIM(dist, g2) - for i in range(nperm): - groupRand = np.random.choice(g2, len(g2), replace=False) - self.r_perm2[i] = oneWayANOSIM(dist, groupRand) - self.p_val = [np.mean(self.r_perm1 > self.R_obs1), np.mean(self.r_perm2 > self.R_obs2)] - if factor2 is not None and nested: - g1 = np.array(factor1) - g2 = np.array(factor2) - comb = np.array(zip(g1, g2), dtype=[('group1', 'S10'), ('group2', 'S10')]) - sortIX = comb.argsort(order='group2') - comb = comb[sortIX] - dist1 = dist[sortIX,:][:,sortIX] - gpR = [] - for i in np.unique(comb['group2']): - withinMat = dist1[comb['group2']==i,:][:,comb['group2']==i] - gpR.append(oneWayANOSIM(withinMat, comb['group1'][comb['group2']==i])) - self.R_obs1 = np.mean(gpR) - comb2 = comb.copy() - for i in range(nperm): - permG = [] - for j in np.unique(comb2['group2']): - gPerm = np.random.choice(comb2['group1'][comb2['group2']==j], np.sum(comb2['group2']==j), replace=False) - permG.extend(gPerm) - comb2['group1'] = permG - gpR = [] - for k in np.unique(comb2['group2']): - withinMat = dist1[comb['group2']==k,:][:,comb['group2']==k] - gpR.append(oneWayANOSIM(withinMat, comb2['group1'][comb2['group2']==k])) - self.r_perm1[i] = np.mean(gpR) - dist2 = dist1.copy() - li = np.tril_indices(dist1.shape[0]) - dist2[li] = np.nan - rankMat = dist2.flatten().argsort().argsort().reshape(dist2.shape) - uniqueSites = np.unique(comb)[np.unique(comb).argsort(order='group2')]['group1'] - collapseMat =np.zeros((len(uniqueSites), len(uniqueSites))) - for i in range(len(uniqueSites)): - for j in range(len(uniqueSites)): - collapseMat[i,j] = np.mean(rankMat[comb['group1']==uniqueSites[i],:][:,comb['group1']==uniqueSites[j]]) - np.fill_diagonal(collapseMat, 0) - collapseGroup = np.unique(comb)['group2'] - self.R_obs2 = oneWayANOSIM(collapseMat, collapseGroup) - for i in range(nperm): - groupRand = np.random.choice(collapseGroup, len(collapseGroup), replace=False) - self.r_perm2[i] = oneWayANOSIM(collapseMat, groupRand) - self.p_val = [np.mean(self.r_perm1 > self.R_obs1), np.mean(self.r_perm2 > self.R_obs2)] - self.perm = nperm + """ + + def __init__(self, dist, factor1, factor2=None, nested=False, nperm=999): + if isinstance(dist, DataFrame): + dist = np.array(dist) + if dist.shape[0] != dist.shape[1]: + msg = "Matrix dist must be a square, symmetric distance matrix" + raise ValueError(msg) + if not np.allclose(dist.T, dist): + msg = "Matrix dist must be a square, symmetric distance matrix" + raise ValueError(msg) + if np.any(dist < 0): + msg = "Distance matrix cannot have negative values" + raise ValueError(msg) + self.r_perm1 = np.empty(nperm) + self.r_perm2 = np.empty(nperm) + self.R_obs1 = None + self.R_obs2 = None + if factor2 is None: + g1 = np.array(factor1) + self.R_obs1 = oneWayANOSIM(dist, g1) + for i in range(nperm): + groupRand = np.random.choice(g1, len(g1), replace=False) + self.r_perm1[i] = oneWayANOSIM(dist, groupRand) + self.p_val = np.mean(self.r_perm1 > self.R_obs1) + if factor2 is not None and not nested: + g1 = np.array(factor1) + self.R_obs1 = oneWayANOSIM(dist, g1) + for i in range(nperm): + groupRand = np.random.choice(g1, len(g1), replace=False) + self.r_perm1[i] = oneWayANOSIM(dist, groupRand) + g2 = np.array(factor2) + self.R_obs2 = oneWayANOSIM(dist, g2) + for i in range(nperm): + groupRand = np.random.choice(g2, len(g2), replace=False) + self.r_perm2[i] = oneWayANOSIM(dist, groupRand) + self.p_val = [ + np.mean(self.r_perm1 > self.R_obs1), + np.mean(self.r_perm2 > self.R_obs2), + ] + if factor2 is not None and nested: + g1 = np.array(factor1) + g2 = np.array(factor2) + comb = np.array(zip(g1, g2), dtype=[("group1", "S10"), ("group2", "S10")]) + sortIX = comb.argsort(order="group2") + comb = comb[sortIX] + dist1 = dist[sortIX, :][:, sortIX] + gpR = [] + for i in np.unique(comb["group2"]): + withinMat = dist1[comb["group2"] == i, :][:, comb["group2"] == i] + gpR.append(oneWayANOSIM(withinMat, comb["group1"][comb["group2"] == i])) + self.R_obs1 = np.mean(gpR) + comb2 = comb.copy() + for i in range(nperm): + permG = [] + for j in np.unique(comb2["group2"]): + gPerm = np.random.choice( + comb2["group1"][comb2["group2"] == j], + np.sum(comb2["group2"] == j), + replace=False, + ) + permG.extend(gPerm) + comb2["group1"] = permG + gpR = [] + for k in np.unique(comb2["group2"]): + withinMat = dist1[comb["group2"] == k, :][:, comb["group2"] == k] + gpR.append( + oneWayANOSIM(withinMat, comb2["group1"][comb2["group2"] == k]) + ) + self.r_perm1[i] = np.mean(gpR) + dist2 = dist1.copy() + li = np.tril_indices(dist1.shape[0]) + dist2[li] = np.nan + rankMat = dist2.flatten().argsort().argsort().reshape(dist2.shape) + uniqueSites = np.unique(comb)[np.unique(comb).argsort(order="group2")][ + "group1" + ] + collapseMat = np.zeros((len(uniqueSites), len(uniqueSites))) + for i in range(len(uniqueSites)): + for j in range(len(uniqueSites)): + collapseMat[i, j] = np.mean( + rankMat[comb["group1"] == uniqueSites[i], :][ + :, comb["group1"] == uniqueSites[j] + ] + ) + np.fill_diagonal(collapseMat, 0) + collapseGroup = np.unique(comb)["group2"] + self.R_obs2 = oneWayANOSIM(collapseMat, collapseGroup) + for i in range(nperm): + groupRand = np.random.choice( + collapseGroup, len(collapseGroup), replace=False + ) + self.r_perm2[i] = oneWayANOSIM(collapseMat, groupRand) + self.p_val = [ + np.mean(self.r_perm1 > self.R_obs1), + np.mean(self.r_perm2 > self.R_obs2), + ] + self.perm = nperm + + def summary(self): + if self.R_obs2 is None: + summ1 = "\nANOSIM\nObserved R = {0:.3}\np-value = {1:.3}\n{2} permutations".format( + self.R_obs1, self.p_val, self.perm + ) + return summ1 + else: + summ1 = "\nANOSIM: Factor 1\nObserved R = {0:.3}\np-value = {1:.3}\n{2} permutations".format( + self.R_obs1, self.p_val[0], self.perm + ) + summ2 = "\nANOSIM: Factor 2\nObserved R = {0:.3}\np-value = {1:.3}\n{2} permutations".format( + self.R_obs2, self.p_val[1], self.perm + ) + return summ1 + "\n" + summ2 + def plot(self): + if self.R_obs2 is None: + f, ax = plt.subplots() + ax.hist( + self.r_perm1, + 50, + normed=1, + color="blue", + alpha=0.5, + histtype="stepfilled", + linewidth=1, + ) + ax.axvline( + self.R_obs1, + linewidth=2, + linestyle="dashed", + color="red", + label="Observed R", + ) + ax.set_ylabel("Density") + ax.set_xlabel("R-statistic") + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + ax.yaxis.set_ticks_position("left") + ax.xaxis.set_ticks_position("bottom") + ax.legend(loc=1) + plt.show() + else: + f, ax = plt.subplots(2, 1, figsize=(6.5, 8.5)) + ax[0].hist( + self.r_perm1, + 50, + normed=1, + color="blue", + alpha=0.5, + histtype="stepfilled", + linewidth=1, + ) + ax[0].axvline( + self.R_obs1, + linewidth=2, + linestyle="dashed", + color="red", + label="Observed R", + ) + ax[0].set_ylabel("Density") + ax[0].set_xlabel("R-statistic") + ax[0].spines["top"].set_visible(False) + ax[0].spines["right"].set_visible(False) + ax[0].yaxis.set_ticks_position("left") + ax[0].xaxis.set_ticks_position("bottom") + ax[0].legend(loc=1) + ax[0].set_title("Factor 1") - def summary(self): - if self.R_obs2 is None: - summ1 = '\nANOSIM\nObserved R = {0:.3}\np-value = {1:.3}\n{2} permutations'.format(self.R_obs1, self.p_val, self.perm) - return summ1 - else: - summ1 = '\nANOSIM: Factor 1\nObserved R = {0:.3}\np-value = {1:.3}\n{2} permutations'.format(self.R_obs1, self.p_val[0], self.perm) - summ2 = '\nANOSIM: Factor 2\nObserved R = {0:.3}\np-value = {1:.3}\n{2} permutations'.format(self.R_obs2, self.p_val[1], self.perm) - return summ1 + '\n' + summ2 - def plot(self): - if self.R_obs2 is None: - f, ax = plt.subplots() - ax.hist(self.r_perm1, 50, normed=1, color='blue', alpha=0.5, histtype='stepfilled', linewidth=1) - ax.axvline(self.R_obs1, linewidth=2, linestyle='dashed', color='red', label='Observed R') - ax.set_ylabel("Density") - ax.set_xlabel("R-statistic") - ax.spines['top'].set_visible(False) - ax.spines['right'].set_visible(False) - ax.yaxis.set_ticks_position('left') - ax.xaxis.set_ticks_position('bottom') - ax.legend(loc=1) - plt.show() - else: - f, ax = plt.subplots(2, 1, figsize=(6.5, 8.5)) - ax[0].hist(self.r_perm1, 50, normed=1, color='blue', alpha=0.5, histtype='stepfilled', linewidth=1) - ax[0].axvline(self.R_obs1, linewidth=2, linestyle='dashed', color='red', label='Observed R') - ax[0].set_ylabel("Density") - ax[0].set_xlabel("R-statistic") - ax[0].spines['top'].set_visible(False) - ax[0].spines['right'].set_visible(False) - ax[0].yaxis.set_ticks_position('left') - ax[0].xaxis.set_ticks_position('bottom') - ax[0].legend(loc=1) - ax[0].set_title('Factor 1') + ax[1].hist( + self.r_perm2, + 50, + normed=1, + color="red", + alpha=0.5, + histtype="stepfilled", + linewidth=1, + ) + ax[1].axvline( + self.R_obs2, + linewidth=2, + linestyle="dashed", + color="red", + label="Observed R", + ) + ax[1].set_ylabel("Density") + ax[1].set_xlabel("R-statistic") + ax[1].spines["top"].set_visible(False) + ax[1].spines["right"].set_visible(False) + ax[1].yaxis.set_ticks_position("left") + ax[1].xaxis.set_ticks_position("bottom") + ax[1].legend(loc=1) + ax[1].set_title("Factor 2") + plt.show() - ax[1].hist(self.r_perm2, 50, normed=1, color='red', alpha=0.5, histtype='stepfilled', linewidth=1) - ax[1].axvline(self.R_obs2, linewidth=2, linestyle='dashed', color='red', label='Observed R') - ax[1].set_ylabel("Density") - ax[1].set_xlabel("R-statistic") - ax[1].spines['top'].set_visible(False) - ax[1].spines['right'].set_visible(False) - ax[1].yaxis.set_ticks_position('left') - ax[1].xaxis.set_ticks_position('bottom') - ax[1].legend(loc=1) - ax[1].set_title('Factor 2') - plt.show() def oneWayANOSIM(x, group): - sortIX = group.argsort() - sorted_G = group[sortIX] - sorted_Dist = x[sortIX,:][:,sortIX] - groupMat = np.array([sorted_G == i for i in sorted_G]) - ui = np.triu_indices(groupMat.shape[0], k=1) - distU = sorted_Dist[ui] - distU = distU.argsort().argsort() - groupU = groupMat[ui] - r_w = np.mean(distU[groupU==True]) - r_b = np.mean(distU[groupU==False]) - n = x.shape[0] - denom = n*(n-1)/4. - return (r_b - r_w) / denom + sortIX = group.argsort() + sorted_G = group[sortIX] + sorted_Dist = x[sortIX, :][:, sortIX] + groupMat = np.array([sorted_G == i for i in sorted_G]) + ui = np.triu_indices(groupMat.shape[0], k=1) + distU = sorted_Dist[ui] + distU = distU.argsort().argsort() + groupU = groupMat[ui] + r_w = np.mean(distU[groupU == True]) + r_b = np.mean(distU[groupU == False]) + n = x.shape[0] + denom = n * (n - 1) / 4.0 + return (r_b - r_w) / denom diff --git a/ecopy/matrix_comp/bioenv.py b/ecopy/matrix_comp/bioenv.py index 35b0403..1753772 100644 --- a/ecopy/matrix_comp/bioenv.py +++ b/ecopy/matrix_comp/bioenv.py @@ -4,8 +4,9 @@ from scipy.stats import spearmanr from itertools import combinations + def bioenv(dist, vars_df, columns=None): - ''' + """ Docstring for function ecopy.simper ==================== Find best subset of environmental variables maximally correlated with @@ -47,33 +48,33 @@ def bioenv(dist, vars_df, columns=None): be = ep.bioenv(dm, df) print(be) - ''' + """ if not isinstance(dist, (np.ndarray, DataFrame)): - msg = 'Must provide a numpy.ndarray or pandas.DataFrame as input' + msg = "Must provide a numpy.ndarray or pandas.DataFrame as input" raise TypeError(msg) if dist.shape[0] != dist.shape[1]: - msg = 'Matrix dist must be a square, symmetric distance matrix' + msg = "Matrix dist must be a square, symmetric distance matrix" raise ValueError(msg) if not np.allclose(dist.T, dist): - msg = 'Matrix dist must be a square, symmetric distance matrix' + msg = "Matrix dist must be a square, symmetric distance matrix" raise ValueError(msg) if np.any(dist < 0): - msg = 'Distance matrix cannot have negative values' + msg = "Distance matrix cannot have negative values" raise ValueError(msg) if not isinstance(vars_df, DataFrame): - msg = 'Must provide a pandas.DataFrame as input' + msg = "Must provide a pandas.DataFrame as input" raise TypeError(msg) if columns is None: columns = vars_df.columns.values.tolist() if len(set(columns)) != len(columns): - msg = 'Duplicate column names are not supported' + msg = "Duplicate column names are not supported" raise ValueError(msg) if len(columns) < 1: - msg = 'Must provide at least one column' + msg = "Must provide at least one column" raise ValueError(msg) for column in columns: @@ -83,11 +84,10 @@ def bioenv(dist, vars_df, columns=None): try: vars_df = vars_df.astype(float) except ValueError: - raise TypeError("All specified columns in the data frame must be " - "numeric.") - + raise TypeError("All specified columns in the data frame must be " "numeric.") + n = len(columns) - ntake = 2**n - 1 + ntake = 2 ** n - 1 if n > 8: print("%i possible subsets (this may take time...)" % ntake) @@ -96,23 +96,22 @@ def bioenv(dist, vars_df, columns=None): # columns within a tight loop and using a numpy array ends up being ~2x # faster. vars_array = _scale(vars_df).values - dm_flat = squareform(dist, force='tovector', checks=False) + dm_flat = squareform(dist, force="tovector", checks=False) num_vars = len(columns) var_idxs = np.arange(num_vars) # For each subset size, store the best combination of variables: # (string identifying best vars, subset size, rho) - max_rhos = np.empty(num_vars, dtype=[('vars', object), - ('size', int), - ('correlation', float)]) + max_rhos = np.empty( + num_vars, dtype=[("vars", object), ("size", int), ("correlation", float)] + ) for subset_size in range(1, num_vars + 1): max_rho = None for subset_idxs in combinations(var_idxs, subset_size): # Compute Euclidean distances using the current subset of # variables. pdist returns the distances in condensed form. - vars_dm_flat = pdist(vars_array[:, subset_idxs], - metric='euclidean') + vars_dm_flat = pdist(vars_array[:, subset_idxs], metric="euclidean") rho = spearmanr(dm_flat, vars_dm_flat)[0] # If there are ties for the best rho at a given subset size, choose @@ -120,10 +119,11 @@ def bioenv(dist, vars_df, columns=None): if max_rho is None or rho > max_rho[0]: max_rho = (rho, subset_idxs) - vars_label = ', '.join([columns[i] for i in max_rho[1]]) + vars_label = ", ".join([columns[i] for i in max_rho[1]]) max_rhos[subset_size - 1] = (vars_label, subset_size, max_rho[0]) - return DataFrame.from_records(max_rhos, index='vars') + return DataFrame.from_records(max_rhos, index="vars") + def _scale(df): df = df.copy() @@ -131,6 +131,8 @@ def _scale(df): df /= df.std() if df.isnull().any().any(): - raise ValueError('Column(s) in the data frame could not be scaled, ' - 'likely because the column(s) had no variance.') + raise ValueError( + "Column(s) in the data frame could not be scaled, " + "likely because the column(s) had no variance." + ) return df diff --git a/ecopy/matrix_comp/cca.py b/ecopy/matrix_comp/cca.py index 3c03001..17c5a66 100644 --- a/ecopy/matrix_comp/cca.py +++ b/ecopy/matrix_comp/cca.py @@ -3,8 +3,9 @@ import matplotlib.pyplot as plt from ecopy import ca + class cca(object): - """ + """ Docstring for function ecopy.cca ==================== Conducts canonical correspondance analysis (CA) for a given @@ -62,206 +63,282 @@ class cca(object): print(cca_fit.anova()) cca_fit.triplot() """ - def __init__(self, Y, X, varNames_y=None, varNames_x=None, rowNames=None, scaling=1): - if not isinstance(Y, (DataFrame, np.ndarray)): - msg = 'Matrix Y must be a pandas.DataFrame or numpy.ndarray' - raise ValueError(msg) - if not isinstance(X, (DataFrame, np.ndarray)): - msg = 'Matrix X must be a pandas.DataFrame or numpy.ndarray' - raise ValueError(msg) - if isinstance(X, DataFrame): - if X.isnull().any().any(): - msg = 'Matrix X contains null values' - raise ValueError(msg) - if isinstance(X, np.ndarray): - if X.dtype=='object': - msg = 'Matrix X cannot be a numpy.ndarray with object dtype' - raise ValueError(msg) - if np.isnan(X).any(): - msg = 'Matrix X contains null values' - raise ValueError(msg) - if isinstance(Y, DataFrame): - if Y.isnull().any().any(): - msg = 'Matrix Y contains null values' - raise ValueError(msg) - if (Y.dtypes == 'object').any(): - msg = 'Matrix Y can only contain numeric values' - raise ValueError(msg) - if isinstance(Y, np.ndarray): - if np.isnan(Y).any(): - msg = 'Matrix Y contains null values' - raise ValueError(msg) - if varNames_y is None: - if isinstance(Y, DataFrame): - varNames_y = Y.columns - elif isinstance(Y, np.ndarray): - varNames_y = ['Sp {0}'.format(x) for x in range(1, Y.shape[1]+1)] - if varNames_x is None: - if isinstance(X, DataFrame): - varNames_x = X.columns - elif isinstance(X, np.ndarray): - varNames_x = ['Pred {0}'.format(x) for x in range(1, X.shape[1]+1)] - if rowNames is None: - if isinstance(Y, DataFrame): - rowNames = Y.index.values - elif isinstance(Y, np.ndarray): - rowNames = ['Site {0}'.format(x) for x in range(1, Y.shape[0]+1)] - if scaling not in [1,2]: - msg = 'Scaling must be 1 or 2' - raise ValueError(msg) - tolerance = 1E-6 - y_mat = np.array(Y, dtype='float') - if isinstance(X, np.ndarray): - x_mat = X - elif isinstance(X, DataFrame): - x_mat = np.array(get_dummies(X), dtype='float') - nrow = X.shape[0] - ncol_x = X.shape[1] - ncol_y = Y.shape[1] - tot = y_mat.sum() - self.r_w = y_mat.sum(axis=1).reshape(1, nrow) / tot - self.c_w = y_mat.sum(axis=0).reshape(ncol_y, 1) / tot - x_mu = self.r_w.dot(x_mat) - x_mu = x_mu.reshape(1, ncol_x) - ones = np.ones(nrow).reshape(nrow, 1) - mu_mat = ones.dot(x_mu) - D = x_mat - mu_mat - sd = np.sqrt(self.r_w.dot(D**2)) - scale_mat = np.diag(1/sd.flatten()) - x_scale = D.dot(scale_mat) - O_mat = y_mat / tot - F_mat = self.r_w.T.dot(self.c_w.T) - Q_mat = (O_mat - F_mat) / np.sqrt(F_mat) - W = np.diag(self.r_w.flatten()) - B = np.linalg.pinv(x_scale.T.dot(W).dot(x_scale)).dot(x_scale.T).dot(W**0.5).dot(Q_mat) - Yhat = (W**0.5).dot(x_scale).dot(B) - Syy = Yhat.T.dot(Yhat) - evals, evecs = np.linalg.eig(Syy) - idx = evals.argsort()[::-1] - self.evals = np.real(evals[idx]) - self.U = np.real(evecs[:,idx]) - self.U = self.U[:,self.evals>tolerance] - self.evals = self.evals[self.evals>tolerance] - Uhat = Q_mat.dot(self.U).dot(np.diag(self.evals**-0.5)) - Wcol = np.diag(self.c_w.flatten()**-0.5) - Wrow = np.diag(self.r_w.flatten()**-0.5) - V = Wcol.dot(self.U) - Vhat = Wrow.dot(Uhat) - F = Vhat.dot(np.diag(self.evals**0.5)) - Fhat = V.dot(np.diag(self.evals**0.5)) - CAlist = ['CA Axis {0}'.format(i) for i in range(1, len(self.evals)+1)] - self.resid = np.array(DataFrame(Q_mat - Yhat, columns=varNames_y, index=rowNames)) - self.res_evals, self.res_evecs = np.linalg.eig(self.resid.T.dot(self.resid)) - self.res_evals = np.real(self.res_evals[self.res_evals>1E-9]) - self.y_mat = y_mat - self.x_mat = x_mat - if scaling==1: - Wrow = np.diag(self.r_w.flatten()**0.5) - self.spScores = DataFrame(V, columns=CAlist, index=varNames_y) - self.siteScores = DataFrame(F, columns=CAlist, index=rowNames) - self.siteFitted = DataFrame(Wrow.dot(Yhat).dot(self.U), columns=CAlist, index=rowNames) - mu_z = self.r_w.dot(self.siteFitted) - mu_z_mat = ones.dot(mu_z) - D_z = self.siteFitted - mu_z_mat - sd_z = np.sqrt(self.r_w.dot(D_z**2)) - scale_z_mat = np.diag(1/sd_z.flatten()) - scaled_Z= D_z.dot(scale_z_mat) - self.varScores = DataFrame(x_scale.T.dot(Wrow).dot(scaled_Z).dot(np.diag(self.evals**0.5)), columns=CAlist, index=varNames_x) - if scaling==2: - Wrow = np.diag(self.r_w.flatten()**0.5) - self.spScores = DataFrame(Fhat, columns=CAlist, index=varNames_y) - self.siteScores = DataFrame(Vhat, columns=CAlist, index=rowNames) - self.siteFitted = DataFrame(Wrow.dot(Yhat).dot(self.U).dot(np.diag(self.evals**-0.5)), columns=CAlist, index=rowNames) - mu_z = self.r_w.dot(self.siteFitted) - mu_z_mat = ones.dot(mu_z) - D_z = self.siteFitted - mu_z_mat - sd_z = np.sqrt(self.r_w.dot(D_z**2)) - scale_z_mat = np.diag(1/sd_z.flatten()) - scaled_Z= D_z.dot(scale_z_mat) - self.varScores = DataFrame(x_scale.T.dot(Wrow).dot(scaled_Z), columns=CAlist, index=varNames_x) - - def summary(self): - print('Constrained variance = {0:.3}'.format(np.sum(self.evals))) - print('Unconstrained varience = {0:.3}'.format(np.sum(self.res_evals))) - names = ['CCA {0}'.format(x) for x in range(1, len(self.evals)+1)] - data = np.vstack((np.round(self.evals, 3), np.round(self.evals/self.evals.sum(),3))) - SumTable1 = DataFrame(data, index = ['Variance', 'Prop. Variance'], columns=names) - print('Constrained Axes') - print(SumTable1) - print('\n') - names2 = ['CA {0}'.format(x) for x in range(1, len(self.res_evals)+1)] - data2 = np.vstack((np.round(self.res_evals, 3), np.round(self.res_evals/self.res_evals.sum(),3))) - SumTable2 = DataFrame(data2, index = ['Variance', 'Prop. Variance'], columns=names2) - print('Unconstrained Axes') - print(SumTable2) - - def anova(self, nperm=999): - constrained = np.sum(self.evals) - unconstrained = np.sum(self.res_evals) - Fobs = (constrained/len(self.evals)) / (unconstrained/len(self.res_evals)) - Fperm = np.empty(nperm) - for i in range(nperm): - n = self.y_mat.shape[0] - nrow = n - ncol_y = self.y_mat.shape[1] - ncol_x = self.x_mat.shape[1] - idx = np.random.choice(n, n, replace=False) - y_perm = self.y_mat[idx,:] - tot = y_perm.sum() - r_w = y_perm.sum(axis=1).reshape(1, nrow) / tot - c_w = y_perm.sum(axis=0).reshape(ncol_y, 1) / tot - x_mu = r_w.dot(self.x_mat) - x_mu = x_mu.reshape(1, ncol_x) - ones = np.ones(nrow).reshape(nrow, 1) - mu_mat = ones.dot(x_mu) - D = self.x_mat - mu_mat - sd = np.sqrt(r_w.dot(D**2)) - scale_mat = np.diag(1/sd.flatten()) - x_scale = D.dot(scale_mat) - O_mat = y_perm / tot - F_mat = r_w.T.dot(c_w.T) - Q_mat = (O_mat - F_mat) / np.sqrt(F_mat) - W = np.diag(r_w.flatten()) - B = np.linalg.pinv(x_scale.T.dot(W).dot(x_scale)).dot(x_scale.T).dot(W**0.5).dot(Q_mat) - Yhat = (W**0.5).dot(x_scale).dot(B) - Syy = Yhat.T.dot(Yhat) - evals_perm, evecs = np.linalg.eig(Syy) - evals_perm = np.real(evals_perm[evals_perm.argsort()[::-1]]) - evals_perm = evals_perm[evals_perm>1E-6] - resid = np.array(DataFrame(Q_mat - Yhat)) - res_evals, res_evecs = np.linalg.eig(resid.T.dot(resid)) - res_evals = np.real(res_evals[res_evals>1E-6]) - c_perm = np.sum(evals_perm) - u_perm = np.sum(res_evals) - Fperm[i] = (c_perm/len(evals_perm)) / (u_perm/len(res_evals)) - print('Model F-statistic = {0:.3}'.format(Fobs)) - print('p = {0:.4}'.format(np.mean(Fperm > Fobs))) - - - def triplot(self, xax=1, yax=2): - xplot = xax-1 - yplot = yax-1 - f, ax = plt.subplots() - for i in range(self.spScores.shape[0]): - ax.plot(self.spScores.iloc[i,xplot], self.spScores.iloc[i,yplot], ms=0) - ax.text(self.spScores.iloc[i,xplot], self.spScores.iloc[i,yplot], self.spScores.index.values[i], color='r', ha='center', va='center') - for i in range(self.siteScores.shape[0]): - ax.plot(self.siteScores.iloc[i,xplot], self.siteScores.iloc[i,yplot], ms=0) - ax.text(self.siteScores.iloc[i,xplot], self.siteScores.iloc[i,yplot], self.siteScores.index.values[i], color='k', ha='center', va='center') - for i in range(self.varScores.shape[0]): - ax.arrow(0, 0, self.varScores.iloc[i, xplot], self.varScores.iloc[i, yplot], color='b', head_width=0.1) - ax.text(self.varScores.iloc[i,xplot]*1.2, self.varScores.iloc[i,yplot]*1.2, self.varScores.index.values[i], color='b', ha='center', va='center') - xmins = (self.spScores.iloc[:,xplot].min(), self.siteScores.iloc[:,xplot].min(), self.varScores.iloc[:,xplot].min()*1.2) - xmax = (self.spScores.iloc[:,xplot].max(), self.siteScores.iloc[:,xplot].max(), self.varScores.iloc[:,xplot].max()*1.2) - ymins = (self.spScores.iloc[:,yplot].min(), self.siteScores.iloc[:,yplot].min(), self.varScores.iloc[:,yplot].min()*1.2) - ymax = (self.spScores.iloc[:,yplot].max(), self.siteScores.iloc[:,yplot].max(), self.varScores.iloc[:,yplot].max()*1.2) - ax.set_xlabel('CA {0}'.format(xax)) - ax.set_ylabel('CA {0}'.format(yax)) - ax.set_xlim([min(xmins), max(xmax)]) - ax.set_ylim([min(ymins), max(ymax)]) - plt.show() - + def __init__( + self, Y, X, varNames_y=None, varNames_x=None, rowNames=None, scaling=1 + ): + if not isinstance(Y, (DataFrame, np.ndarray)): + msg = "Matrix Y must be a pandas.DataFrame or numpy.ndarray" + raise ValueError(msg) + if not isinstance(X, (DataFrame, np.ndarray)): + msg = "Matrix X must be a pandas.DataFrame or numpy.ndarray" + raise ValueError(msg) + if isinstance(X, DataFrame): + if X.isnull().any().any(): + msg = "Matrix X contains null values" + raise ValueError(msg) + if isinstance(X, np.ndarray): + if X.dtype == "object": + msg = "Matrix X cannot be a numpy.ndarray with object dtype" + raise ValueError(msg) + if np.isnan(X).any(): + msg = "Matrix X contains null values" + raise ValueError(msg) + if isinstance(Y, DataFrame): + if Y.isnull().any().any(): + msg = "Matrix Y contains null values" + raise ValueError(msg) + if (Y.dtypes == "object").any(): + msg = "Matrix Y can only contain numeric values" + raise ValueError(msg) + if isinstance(Y, np.ndarray): + if np.isnan(Y).any(): + msg = "Matrix Y contains null values" + raise ValueError(msg) + if varNames_y is None: + if isinstance(Y, DataFrame): + varNames_y = Y.columns + elif isinstance(Y, np.ndarray): + varNames_y = ["Sp {0}".format(x) for x in range(1, Y.shape[1] + 1)] + if varNames_x is None: + if isinstance(X, DataFrame): + varNames_x = X.columns + elif isinstance(X, np.ndarray): + varNames_x = ["Pred {0}".format(x) for x in range(1, X.shape[1] + 1)] + if rowNames is None: + if isinstance(Y, DataFrame): + rowNames = Y.index.values + elif isinstance(Y, np.ndarray): + rowNames = ["Site {0}".format(x) for x in range(1, Y.shape[0] + 1)] + if scaling not in [1, 2]: + msg = "Scaling must be 1 or 2" + raise ValueError(msg) + tolerance = 1e-6 + y_mat = np.array(Y, dtype="float") + if isinstance(X, np.ndarray): + x_mat = X + elif isinstance(X, DataFrame): + x_mat = np.array(get_dummies(X), dtype="float") + nrow = X.shape[0] + ncol_x = X.shape[1] + ncol_y = Y.shape[1] + tot = y_mat.sum() + self.r_w = y_mat.sum(axis=1).reshape(1, nrow) / tot + self.c_w = y_mat.sum(axis=0).reshape(ncol_y, 1) / tot + x_mu = self.r_w.dot(x_mat) + x_mu = x_mu.reshape(1, ncol_x) + ones = np.ones(nrow).reshape(nrow, 1) + mu_mat = ones.dot(x_mu) + D = x_mat - mu_mat + sd = np.sqrt(self.r_w.dot(D ** 2)) + scale_mat = np.diag(1 / sd.flatten()) + x_scale = D.dot(scale_mat) + O_mat = y_mat / tot + F_mat = self.r_w.T.dot(self.c_w.T) + Q_mat = (O_mat - F_mat) / np.sqrt(F_mat) + W = np.diag(self.r_w.flatten()) + B = ( + np.linalg.pinv(x_scale.T.dot(W).dot(x_scale)) + .dot(x_scale.T) + .dot(W ** 0.5) + .dot(Q_mat) + ) + Yhat = (W ** 0.5).dot(x_scale).dot(B) + Syy = Yhat.T.dot(Yhat) + evals, evecs = np.linalg.eig(Syy) + idx = evals.argsort()[::-1] + self.evals = np.real(evals[idx]) + self.U = np.real(evecs[:, idx]) + self.U = self.U[:, self.evals > tolerance] + self.evals = self.evals[self.evals > tolerance] + Uhat = Q_mat.dot(self.U).dot(np.diag(self.evals ** -0.5)) + Wcol = np.diag(self.c_w.flatten() ** -0.5) + Wrow = np.diag(self.r_w.flatten() ** -0.5) + V = Wcol.dot(self.U) + Vhat = Wrow.dot(Uhat) + F = Vhat.dot(np.diag(self.evals ** 0.5)) + Fhat = V.dot(np.diag(self.evals ** 0.5)) + CAlist = ["CA Axis {0}".format(i) for i in range(1, len(self.evals) + 1)] + self.resid = np.array( + DataFrame(Q_mat - Yhat, columns=varNames_y, index=rowNames) + ) + self.res_evals, self.res_evecs = np.linalg.eig(self.resid.T.dot(self.resid)) + self.res_evals = np.real(self.res_evals[self.res_evals > 1e-9]) + self.y_mat = y_mat + self.x_mat = x_mat + if scaling == 1: + Wrow = np.diag(self.r_w.flatten() ** 0.5) + self.spScores = DataFrame(V, columns=CAlist, index=varNames_y) + self.siteScores = DataFrame(F, columns=CAlist, index=rowNames) + self.siteFitted = DataFrame( + Wrow.dot(Yhat).dot(self.U), columns=CAlist, index=rowNames + ) + mu_z = self.r_w.dot(self.siteFitted) + mu_z_mat = ones.dot(mu_z) + D_z = self.siteFitted - mu_z_mat + sd_z = np.sqrt(self.r_w.dot(D_z ** 2)) + scale_z_mat = np.diag(1 / sd_z.flatten()) + scaled_Z = D_z.dot(scale_z_mat) + self.varScores = DataFrame( + x_scale.T.dot(Wrow).dot(scaled_Z).dot(np.diag(self.evals ** 0.5)), + columns=CAlist, + index=varNames_x, + ) + if scaling == 2: + Wrow = np.diag(self.r_w.flatten() ** 0.5) + self.spScores = DataFrame(Fhat, columns=CAlist, index=varNames_y) + self.siteScores = DataFrame(Vhat, columns=CAlist, index=rowNames) + self.siteFitted = DataFrame( + Wrow.dot(Yhat).dot(self.U).dot(np.diag(self.evals ** -0.5)), + columns=CAlist, + index=rowNames, + ) + mu_z = self.r_w.dot(self.siteFitted) + mu_z_mat = ones.dot(mu_z) + D_z = self.siteFitted - mu_z_mat + sd_z = np.sqrt(self.r_w.dot(D_z ** 2)) + scale_z_mat = np.diag(1 / sd_z.flatten()) + scaled_Z = D_z.dot(scale_z_mat) + self.varScores = DataFrame( + x_scale.T.dot(Wrow).dot(scaled_Z), columns=CAlist, index=varNames_x + ) + def summary(self): + print("Constrained variance = {0:.3}".format(np.sum(self.evals))) + print("Unconstrained varience = {0:.3}".format(np.sum(self.res_evals))) + names = ["CCA {0}".format(x) for x in range(1, len(self.evals) + 1)] + data = np.vstack( + (np.round(self.evals, 3), np.round(self.evals / self.evals.sum(), 3)) + ) + SumTable1 = DataFrame(data, index=["Variance", "Prop. Variance"], columns=names) + print("Constrained Axes") + print(SumTable1) + print("\n") + names2 = ["CA {0}".format(x) for x in range(1, len(self.res_evals) + 1)] + data2 = np.vstack( + ( + np.round(self.res_evals, 3), + np.round(self.res_evals / self.res_evals.sum(), 3), + ) + ) + SumTable2 = DataFrame( + data2, index=["Variance", "Prop. Variance"], columns=names2 + ) + print("Unconstrained Axes") + print(SumTable2) + def anova(self, nperm=999): + constrained = np.sum(self.evals) + unconstrained = np.sum(self.res_evals) + Fobs = (constrained / len(self.evals)) / (unconstrained / len(self.res_evals)) + Fperm = np.empty(nperm) + for i in range(nperm): + n = self.y_mat.shape[0] + nrow = n + ncol_y = self.y_mat.shape[1] + ncol_x = self.x_mat.shape[1] + idx = np.random.choice(n, n, replace=False) + y_perm = self.y_mat[idx, :] + tot = y_perm.sum() + r_w = y_perm.sum(axis=1).reshape(1, nrow) / tot + c_w = y_perm.sum(axis=0).reshape(ncol_y, 1) / tot + x_mu = r_w.dot(self.x_mat) + x_mu = x_mu.reshape(1, ncol_x) + ones = np.ones(nrow).reshape(nrow, 1) + mu_mat = ones.dot(x_mu) + D = self.x_mat - mu_mat + sd = np.sqrt(r_w.dot(D ** 2)) + scale_mat = np.diag(1 / sd.flatten()) + x_scale = D.dot(scale_mat) + O_mat = y_perm / tot + F_mat = r_w.T.dot(c_w.T) + Q_mat = (O_mat - F_mat) / np.sqrt(F_mat) + W = np.diag(r_w.flatten()) + B = ( + np.linalg.pinv(x_scale.T.dot(W).dot(x_scale)) + .dot(x_scale.T) + .dot(W ** 0.5) + .dot(Q_mat) + ) + Yhat = (W ** 0.5).dot(x_scale).dot(B) + Syy = Yhat.T.dot(Yhat) + evals_perm, evecs = np.linalg.eig(Syy) + evals_perm = np.real(evals_perm[evals_perm.argsort()[::-1]]) + evals_perm = evals_perm[evals_perm > 1e-6] + resid = np.array(DataFrame(Q_mat - Yhat)) + res_evals, res_evecs = np.linalg.eig(resid.T.dot(resid)) + res_evals = np.real(res_evals[res_evals > 1e-6]) + c_perm = np.sum(evals_perm) + u_perm = np.sum(res_evals) + Fperm[i] = (c_perm / len(evals_perm)) / (u_perm / len(res_evals)) + print("Model F-statistic = {0:.3}".format(Fobs)) + print("p = {0:.4}".format(np.mean(Fperm > Fobs))) + def triplot(self, xax=1, yax=2): + xplot = xax - 1 + yplot = yax - 1 + f, ax = plt.subplots() + for i in range(self.spScores.shape[0]): + ax.plot(self.spScores.iloc[i, xplot], self.spScores.iloc[i, yplot], ms=0) + ax.text( + self.spScores.iloc[i, xplot], + self.spScores.iloc[i, yplot], + self.spScores.index.values[i], + color="r", + ha="center", + va="center", + ) + for i in range(self.siteScores.shape[0]): + ax.plot( + self.siteScores.iloc[i, xplot], self.siteScores.iloc[i, yplot], ms=0 + ) + ax.text( + self.siteScores.iloc[i, xplot], + self.siteScores.iloc[i, yplot], + self.siteScores.index.values[i], + color="k", + ha="center", + va="center", + ) + for i in range(self.varScores.shape[0]): + ax.arrow( + 0, + 0, + self.varScores.iloc[i, xplot], + self.varScores.iloc[i, yplot], + color="b", + head_width=0.1, + ) + ax.text( + self.varScores.iloc[i, xplot] * 1.2, + self.varScores.iloc[i, yplot] * 1.2, + self.varScores.index.values[i], + color="b", + ha="center", + va="center", + ) + xmins = ( + self.spScores.iloc[:, xplot].min(), + self.siteScores.iloc[:, xplot].min(), + self.varScores.iloc[:, xplot].min() * 1.2, + ) + xmax = ( + self.spScores.iloc[:, xplot].max(), + self.siteScores.iloc[:, xplot].max(), + self.varScores.iloc[:, xplot].max() * 1.2, + ) + ymins = ( + self.spScores.iloc[:, yplot].min(), + self.siteScores.iloc[:, yplot].min(), + self.varScores.iloc[:, yplot].min() * 1.2, + ) + ymax = ( + self.spScores.iloc[:, yplot].max(), + self.siteScores.iloc[:, yplot].max(), + self.varScores.iloc[:, yplot].max() * 1.2, + ) + ax.set_xlabel("CA {0}".format(xax)) + ax.set_ylabel("CA {0}".format(yax)) + ax.set_xlim([min(xmins), max(xmax)]) + ax.set_ylim([min(ymins), max(ymax)]) + plt.show() diff --git a/ecopy/matrix_comp/ccor.py b/ecopy/matrix_comp/ccor.py index 9c5030a..50cfa76 100644 --- a/ecopy/matrix_comp/ccor.py +++ b/ecopy/matrix_comp/ccor.py @@ -2,8 +2,9 @@ from pandas import DataFrame import matplotlib.pyplot as plt + class ccor(object): - """ + """ Docstring for function ecopy.ccor ==================== Conducts canonical correlation analysis for two matrices @@ -54,118 +55,143 @@ class ccor(object): cc.summary() cc.biplot() """ - def __init__(self, Y1, Y2, varNames_1=None, varNames_2=None, stand_1=False, stand_2=False, siteNames=None): - if not isinstance(Y1, (DataFrame, np.ndarray)): - msg = 'Matrix Y1 must be a pandas.DataFrame or numpy.ndarray' - raise ValueError(msg) - if not isinstance(Y2, (DataFrame, np.ndarray)): - msg = 'Matrix Y2 must be a pandas.DataFrame or numpy.ndarray' - raise ValueError(msg) - if isinstance(Y2, DataFrame): - if Y2.isnull().any().any(): - msg = 'Matrix Y2 contains null values' - raise ValueError(msg) - if isinstance(Y2, np.ndarray): - if Y2.dtype=='object': - msg = 'Matrix Y2 cannot be a numpy.ndarray with object dtype' - raise ValueError(msg) - if np.isnan(Y2).any(): - msg = 'Matrix Y2 contains null values' - raise ValueError(msg) - if isinstance(Y1, DataFrame): - if Y1.isnull().any().any(): - msg = 'Matrix Y1 contains null values' - raise ValueError(msg) - if (Y1.dtypes == 'object').any(): - msg = 'Matrix Y1 can only contain numeric values' - raise ValueError(msg) - if isinstance(Y1, np.ndarray): - if np.isnan(Y1).any(): - msg = 'Matrix Y1 contains null values' - raise ValueError(msg) - if varNames_1 is None: - if isinstance(Y1, DataFrame): - varNames_1 = Y1.columns - elif isinstance(Y1, np.ndarray): - varNames_1 = ['Y1 {0}'.format(x) for x in range(1, Y1.shape[1]+1)] - if varNames_2 is None: - if isinstance(Y2, DataFrame): - varNames_2 = Y2.columns - elif isinstance(Y2, np.ndarray): - varNames_2 = ['Y2 {0}'.format(x) for x in range(1, Y2.shape[1]+1)] - if siteNames is None: - if isinstance(Y1, DataFrame): - siteNames = Y1.index.values - elif isinstance(Y1, np.ndarray): - siteNames = ['Site {0}'.format(x) for x in range(1, Y1.shape[0]+1)] - if Y1.shape[0] != Y2.shape[0]: - msg = 'Matrices must have same number of rows' - raise ValueError(msg) - Y1 = np.array(Y1) - Y2 = np.array(Y2) - if stand_1: - Y1 = (Y1 - Y1.mean(axis=0)) / Y1.std(axis=0) - if stand_2: - Y2 = (Y2 - Y2.mean(axis=0)) / Y2.std(axis=0) - df = float(Y1.shape[0] - 1) - D1 = Y1 - Y1.mean(axis=0) - D2 = Y2 - Y2.mean(axis=0) - S1 = D1.T.dot(D1) * 1./df - S2 = D2.T.dot(D2) * 1./df - S12 = D1.T.dot(D2) * 1./df - Chol1 = np.linalg.pinv(np.linalg.cholesky(S1).T) - Chol2 = np.linalg.pinv(np.linalg.cholesky(S2).T) - K = Chol1.T.dot(S12).dot(Chol2) - V, W, U = np.linalg.svd(K) - U = U.T - CoefY1 = Chol1.dot(V) - CoefY2 = Chol2.dot(U) - self.Scores1 = DataFrame(Y1.dot(CoefY1), index=siteNames) - self.Scores2 = DataFrame(Y2.dot(CoefY2), index=siteNames) - self.loadings1 = np.corrcoef(Y1, self.Scores1, rowvar=0)[:5, 5:] - self.loadings2 = np.corrcoef(Y2, self.Scores2, rowvar=0)[:3, 3:] - axes1 = ['CA Axis {0}'.format(x) for x in range(1, self.loadings1.shape[1]+1)] - self.loadings1 = DataFrame(self.loadings1, index=varNames_1, columns=axes1) - axes2 = ['CA Axis {0}'.format(x) for x in range(1, self.loadings2.shape[1]+1)] - self.loadings2 = DataFrame(self.loadings2, index=varNames_2, columns=axes2) - self.evals = W - - - def summary(self): - print('Constrained variance = {0:.3}'.format(np.sum(self.evals))) - print('Constrained variance explained be each axis') - print([str(i) for i in np.round(self.evals, 3)]) - print('Proportion constrained variance') - print([str(i) for i in np.round(self.evals/self.evals.sum(), 3)]) - - def biplot(self, matrix=1, xax=1, yax=2): - xplot = xax-1 - yplot = yax-1 - scores = self.Scores1 - loadings = self.loadings1 - if matrix==2: - scores = self.Scores2 - loadings = self.loadings2 - varNames = loadings.index.values - siteNames = scores.index.values - f, ax = plt.subplots() - for i in range(scores.shape[0]): - ax.plot(scores.iloc[i,xplot], scores.iloc[i,yplot], ms=0) - ax.text(scores.iloc[i,xplot], scores.iloc[i,yplot], siteNames[i], color='r', ha='center', va='center') - for i in range(loadings.shape[0]): - ax.arrow(0, 0, loadings.iloc[i, xplot], loadings.iloc[i, yplot], color='b', head_width=0.1) - ax.text(loadings.iloc[i,xplot]*1.2, loadings.iloc[i,yplot]*1.2, varNames[i], color='b', ha='center', va='center') - xmins = (scores.iloc[:,xplot].min(), loadings.iloc[:,xplot].min()*1.2) - xmax = (scores.iloc[:,xplot].max(), loadings.iloc[:,xplot].max()*1.2) - ymins = (scores.iloc[:,yplot].min(), loadings.iloc[:,yplot].min()*1.2) - ymax = (scores.iloc[:,yplot].max(), loadings.iloc[:,yplot].max()*1.2) - ax.set_xlabel('CCor {0}'.format(xax)) - ax.set_ylabel('CCor {0}'.format(yax)) - ax.set_xlim([min(xmins), max(xmax)]) - ax.set_ylim([min(ymins), max(ymax)]) - plt.show() - - + def __init__( + self, + Y1, + Y2, + varNames_1=None, + varNames_2=None, + stand_1=False, + stand_2=False, + siteNames=None, + ): + if not isinstance(Y1, (DataFrame, np.ndarray)): + msg = "Matrix Y1 must be a pandas.DataFrame or numpy.ndarray" + raise ValueError(msg) + if not isinstance(Y2, (DataFrame, np.ndarray)): + msg = "Matrix Y2 must be a pandas.DataFrame or numpy.ndarray" + raise ValueError(msg) + if isinstance(Y2, DataFrame): + if Y2.isnull().any().any(): + msg = "Matrix Y2 contains null values" + raise ValueError(msg) + if isinstance(Y2, np.ndarray): + if Y2.dtype == "object": + msg = "Matrix Y2 cannot be a numpy.ndarray with object dtype" + raise ValueError(msg) + if np.isnan(Y2).any(): + msg = "Matrix Y2 contains null values" + raise ValueError(msg) + if isinstance(Y1, DataFrame): + if Y1.isnull().any().any(): + msg = "Matrix Y1 contains null values" + raise ValueError(msg) + if (Y1.dtypes == "object").any(): + msg = "Matrix Y1 can only contain numeric values" + raise ValueError(msg) + if isinstance(Y1, np.ndarray): + if np.isnan(Y1).any(): + msg = "Matrix Y1 contains null values" + raise ValueError(msg) + if varNames_1 is None: + if isinstance(Y1, DataFrame): + varNames_1 = Y1.columns + elif isinstance(Y1, np.ndarray): + varNames_1 = ["Y1 {0}".format(x) for x in range(1, Y1.shape[1] + 1)] + if varNames_2 is None: + if isinstance(Y2, DataFrame): + varNames_2 = Y2.columns + elif isinstance(Y2, np.ndarray): + varNames_2 = ["Y2 {0}".format(x) for x in range(1, Y2.shape[1] + 1)] + if siteNames is None: + if isinstance(Y1, DataFrame): + siteNames = Y1.index.values + elif isinstance(Y1, np.ndarray): + siteNames = ["Site {0}".format(x) for x in range(1, Y1.shape[0] + 1)] + if Y1.shape[0] != Y2.shape[0]: + msg = "Matrices must have same number of rows" + raise ValueError(msg) + Y1 = np.array(Y1) + Y2 = np.array(Y2) + if stand_1: + Y1 = (Y1 - Y1.mean(axis=0)) / Y1.std(axis=0) + if stand_2: + Y2 = (Y2 - Y2.mean(axis=0)) / Y2.std(axis=0) + df = float(Y1.shape[0] - 1) + D1 = Y1 - Y1.mean(axis=0) + D2 = Y2 - Y2.mean(axis=0) + S1 = D1.T.dot(D1) * 1.0 / df + S2 = D2.T.dot(D2) * 1.0 / df + S12 = D1.T.dot(D2) * 1.0 / df + Chol1 = np.linalg.pinv(np.linalg.cholesky(S1).T) + Chol2 = np.linalg.pinv(np.linalg.cholesky(S2).T) + K = Chol1.T.dot(S12).dot(Chol2) + V, W, U = np.linalg.svd(K) + U = U.T + CoefY1 = Chol1.dot(V) + CoefY2 = Chol2.dot(U) + self.Scores1 = DataFrame(Y1.dot(CoefY1), index=siteNames) + self.Scores2 = DataFrame(Y2.dot(CoefY2), index=siteNames) + self.loadings1 = np.corrcoef(Y1, self.Scores1, rowvar=0)[:5, 5:] + self.loadings2 = np.corrcoef(Y2, self.Scores2, rowvar=0)[:3, 3:] + axes1 = ["CA Axis {0}".format(x) for x in range(1, self.loadings1.shape[1] + 1)] + self.loadings1 = DataFrame(self.loadings1, index=varNames_1, columns=axes1) + axes2 = ["CA Axis {0}".format(x) for x in range(1, self.loadings2.shape[1] + 1)] + self.loadings2 = DataFrame(self.loadings2, index=varNames_2, columns=axes2) + self.evals = W + def summary(self): + print("Constrained variance = {0:.3}".format(np.sum(self.evals))) + print("Constrained variance explained be each axis") + print([str(i) for i in np.round(self.evals, 3)]) + print("Proportion constrained variance") + print([str(i) for i in np.round(self.evals / self.evals.sum(), 3)]) + def biplot(self, matrix=1, xax=1, yax=2): + xplot = xax - 1 + yplot = yax - 1 + scores = self.Scores1 + loadings = self.loadings1 + if matrix == 2: + scores = self.Scores2 + loadings = self.loadings2 + varNames = loadings.index.values + siteNames = scores.index.values + f, ax = plt.subplots() + for i in range(scores.shape[0]): + ax.plot(scores.iloc[i, xplot], scores.iloc[i, yplot], ms=0) + ax.text( + scores.iloc[i, xplot], + scores.iloc[i, yplot], + siteNames[i], + color="r", + ha="center", + va="center", + ) + for i in range(loadings.shape[0]): + ax.arrow( + 0, + 0, + loadings.iloc[i, xplot], + loadings.iloc[i, yplot], + color="b", + head_width=0.1, + ) + ax.text( + loadings.iloc[i, xplot] * 1.2, + loadings.iloc[i, yplot] * 1.2, + varNames[i], + color="b", + ha="center", + va="center", + ) + xmins = (scores.iloc[:, xplot].min(), loadings.iloc[:, xplot].min() * 1.2) + xmax = (scores.iloc[:, xplot].max(), loadings.iloc[:, xplot].max() * 1.2) + ymins = (scores.iloc[:, yplot].min(), loadings.iloc[:, yplot].min() * 1.2) + ymax = (scores.iloc[:, yplot].max(), loadings.iloc[:, yplot].max() * 1.2) + ax.set_xlabel("CCor {0}".format(xax)) + ax.set_ylabel("CCor {0}".format(yax)) + ax.set_xlim([min(xmins), max(xmax)]) + ax.set_ylim([min(ymins), max(ymax)]) + plt.show() diff --git a/ecopy/matrix_comp/fourthcorner.py b/ecopy/matrix_comp/fourthcorner.py index c7d0d9f..2c989fd 100644 --- a/ecopy/matrix_comp/fourthcorner.py +++ b/ecopy/matrix_comp/fourthcorner.py @@ -2,8 +2,9 @@ from pandas import DataFrame, factorize from scipy.stats import chisquare + class corner4(object): - """ + """ Docstring for function ecopy.corner4 ==================== Conducts a fourth corner analysis for a trait matrix (Q), @@ -37,220 +38,267 @@ class corner4(object): test1 = ep.corner4(env, sp, traits, nperm=99, p_adjustment='fdr') print(test1.summary()) """ - def __init__(self, R, L, Q, nperm=999, model=1, test='both', p_adjustment=None): - if not isinstance(R, (DataFrame)): - msg = 'Matrix R must be a pandas.DataFrame' - raise ValueError(msg) - if not isinstance(L, (np.ndarray, DataFrame)): - msg = 'Matrix L must be a numpy.ndarray or pandas.DataFrame' - raise ValueError(msg) - if not isinstance(Q, (DataFrame)): - msg = 'Matrix Q must be a pandas.DataFrame' - raise ValueError(msg) - if model not in [1,2,3,4]: - msg = 'model must be 1, 2, 3, or 4' - raise ValueError(msg) - if isinstance(L, DataFrame): - L = np.array(L) - if L.dtype != 'int': - msg = 'Matrix L must contain only integer values of species abundances (no floats).\nConvert matrix to integers before proceeding' - raise ValueError(msg) - if np.any(L < 0): - msg ='Matrix L cannot have negative values' - raise ValueError(msg) - if test not in ['greater', 'lower', 'both']: - msg = 'test argument must be greater, lower, or both' - raise ValueError(msg) - if p_adjustment is not None: - if p_adjustment not in ['bonferroni', 'holm', 'fdr']: - msg = 'p_adjustment must be bonferroni, holm, fdr, or None' - raise ValueError(msg) - pval = [] - obsStat = [] - statType = [] - compName = [] - for i in range(R.shape[1]): - for j in range(Q.shape[1]): - if R.iloc[:,i].dtype not in ['float', 'object']: - msg = 'Environmental data must be float or object' - raise ValueError(msg) - if Q.iloc[:,j].dtype not in ['float', 'object']: - msg = 'Trait data must be float or object' - raise ValueError(msg) - stat_perm = np.zeros(nperm) - comp = '{0} - {1}'.format(R.iloc[:,i].name, Q.iloc[:,j].name) - compName.append(comp) - if R.iloc[:,i].dtype=='float' and Q.iloc[:,i].dtype=='float': - stat_obs = QuantQuant(R.iloc[:,i], L, Q.iloc[:,j]) - iteration = 0 - while iteration < nperm: - Lperm = permuteType(L, model) - stat_perm[iteration] = QuantQuant(R.iloc[:,i], Lperm, Q.iloc[:,j]) - iteration += 1 - obsStat.append(stat_obs) - statType.append('Pearson r') - if test is 'greater': - pval.append(np.mean(stat_perm >= stat_obs)) - if test is 'lower': - pval.append(np.mean(stat_perm <= stat_obs)) - if test is 'both': - pval.append(np.mean(np.abs(stat_perm)>=np.abs(stat_obs))) - if R.iloc[:,i].dtype=='object' and Q.iloc[:,j].dtype=='object': - R2 = factorize(R.iloc[:,i])[0] - Q2 = factorize(Q.iloc[:,j])[0] - n_env = len(R.iloc[:,i].unique()) - n_trait = len(Q.iloc[:,j].unique()) - stat_obs = QualQual(R2, L, Q2, n_env, n_trait) - iteration = 0 - while iteration < nperm: - Lperm = permuteType(L, model) - stat_perm[iteration] = QualQual(R2, Lperm, Q2, n_env, n_trait) - iteration += 1 - obsStat.append(stat_obs) - statType.append('Chi-Squared') - if test is 'greater': - pval.append(np.mean(stat_perm >= stat_obs)) - if test is 'lower': - pval.append(np.mean(stat_perm <= stat_obs)) - if test is 'both': - pval.append(np.mean(np.abs(stat_perm)>=np.abs(stat_obs))) - if R.iloc[:,i].dtype=='object' and Q.iloc[:,j].dtype=='float': - stat_obs = QualQuant(R.iloc[:,i], L, Q.iloc[:,j]) - iteration = 0 - while iteration < nperm: - Lperm = permuteType(L, model) - stat_perm[iteration] = QualQuant(R.iloc[:,i], Lperm, Q.iloc[:,j]) - iteration += 1 - obsStat.append(stat_obs) - statType.append('F') - if test is 'greater': - pval.append(np.mean(stat_perm >= stat_obs)) - if test is 'lower': - pval.append(np.mean(stat_perm <= stat_obs)) - if test is 'both': - pval.append(np.mean(np.abs(stat_perm)>=np.abs(stat_obs))) - if R.iloc[:,i].dtype=='float' and Q.iloc[:,j].dtype=='object': - stat_obs = QuantQual(R.iloc[:,i], L, Q.iloc[:,j]) - iteration = 0 - while iteration < nperm: - Lperm = permuteType(L, model) - stat_perm[iteration] = QuantQual(R.iloc[:,i], Lperm, Q.iloc[:,j]) - iteration += 1 - obsStat.append(stat_obs) - statType.append('F-statistic') - if test is 'greater': - pval.append(np.mean(stat_perm >= stat_obs)) - if test is 'lower': - pval.append(np.mean(stat_perm <= stat_obs)) - if test is 'both': - pval.append(np.mean(np.abs(stat_perm)>=np.abs(stat_obs))) - self.results = DataFrame({'Comparison': compName, 'Statistic': statType, 'Observed Stat': np.round(obsStat, 2), 'p-value': np.round(pval, 3)}) - self.results['tail'] = test - self.nperm = nperm - self.adj = p_adjustment - def summary(self): - if self.adj is None: - print('\nFour Corner Analysis - {0} Permutations\n'.format(self.nperm)) - return self.results[['Comparison', 'Statistic', 'Observed Stat', 'tail', 'p-value']] - if self.adj is not None: - print('\nFour Corner Analysis - {0} Permutations\n{1} Correction\n'.format(self.nperm, self.adj)) - self.results['adjusted p-value'] = p_adjust(self.results['p-value'], self.adj) - return self.results[['Comparison', 'Statistic', 'Observed Stat', 'tail', 'p-value', 'adjusted p-value']] + def __init__(self, R, L, Q, nperm=999, model=1, test="both", p_adjustment=None): + if not isinstance(R, (DataFrame)): + msg = "Matrix R must be a pandas.DataFrame" + raise ValueError(msg) + if not isinstance(L, (np.ndarray, DataFrame)): + msg = "Matrix L must be a numpy.ndarray or pandas.DataFrame" + raise ValueError(msg) + if not isinstance(Q, (DataFrame)): + msg = "Matrix Q must be a pandas.DataFrame" + raise ValueError(msg) + if model not in [1, 2, 3, 4]: + msg = "model must be 1, 2, 3, or 4" + raise ValueError(msg) + if isinstance(L, DataFrame): + L = np.array(L) + if L.dtype != "int": + msg = "Matrix L must contain only integer values of species abundances (no floats).\nConvert matrix to integers before proceeding" + raise ValueError(msg) + if np.any(L < 0): + msg = "Matrix L cannot have negative values" + raise ValueError(msg) + if test not in ["greater", "lower", "both"]: + msg = "test argument must be greater, lower, or both" + raise ValueError(msg) + if p_adjustment is not None: + if p_adjustment not in ["bonferroni", "holm", "fdr"]: + msg = "p_adjustment must be bonferroni, holm, fdr, or None" + raise ValueError(msg) + pval = [] + obsStat = [] + statType = [] + compName = [] + for i in range(R.shape[1]): + for j in range(Q.shape[1]): + if R.iloc[:, i].dtype not in ["float", "object"]: + msg = "Environmental data must be float or object" + raise ValueError(msg) + if Q.iloc[:, j].dtype not in ["float", "object"]: + msg = "Trait data must be float or object" + raise ValueError(msg) + stat_perm = np.zeros(nperm) + comp = "{0} - {1}".format(R.iloc[:, i].name, Q.iloc[:, j].name) + compName.append(comp) + if R.iloc[:, i].dtype == "float" and Q.iloc[:, i].dtype == "float": + stat_obs = QuantQuant(R.iloc[:, i], L, Q.iloc[:, j]) + iteration = 0 + while iteration < nperm: + Lperm = permuteType(L, model) + stat_perm[iteration] = QuantQuant( + R.iloc[:, i], Lperm, Q.iloc[:, j] + ) + iteration += 1 + obsStat.append(stat_obs) + statType.append("Pearson r") + if test is "greater": + pval.append(np.mean(stat_perm >= stat_obs)) + if test is "lower": + pval.append(np.mean(stat_perm <= stat_obs)) + if test is "both": + pval.append(np.mean(np.abs(stat_perm) >= np.abs(stat_obs))) + if R.iloc[:, i].dtype == "object" and Q.iloc[:, j].dtype == "object": + R2 = factorize(R.iloc[:, i])[0] + Q2 = factorize(Q.iloc[:, j])[0] + n_env = len(R.iloc[:, i].unique()) + n_trait = len(Q.iloc[:, j].unique()) + stat_obs = QualQual(R2, L, Q2, n_env, n_trait) + iteration = 0 + while iteration < nperm: + Lperm = permuteType(L, model) + stat_perm[iteration] = QualQual(R2, Lperm, Q2, n_env, n_trait) + iteration += 1 + obsStat.append(stat_obs) + statType.append("Chi-Squared") + if test is "greater": + pval.append(np.mean(stat_perm >= stat_obs)) + if test is "lower": + pval.append(np.mean(stat_perm <= stat_obs)) + if test is "both": + pval.append(np.mean(np.abs(stat_perm) >= np.abs(stat_obs))) + if R.iloc[:, i].dtype == "object" and Q.iloc[:, j].dtype == "float": + stat_obs = QualQuant(R.iloc[:, i], L, Q.iloc[:, j]) + iteration = 0 + while iteration < nperm: + Lperm = permuteType(L, model) + stat_perm[iteration] = QualQuant( + R.iloc[:, i], Lperm, Q.iloc[:, j] + ) + iteration += 1 + obsStat.append(stat_obs) + statType.append("F") + if test is "greater": + pval.append(np.mean(stat_perm >= stat_obs)) + if test is "lower": + pval.append(np.mean(stat_perm <= stat_obs)) + if test is "both": + pval.append(np.mean(np.abs(stat_perm) >= np.abs(stat_obs))) + if R.iloc[:, i].dtype == "float" and Q.iloc[:, j].dtype == "object": + stat_obs = QuantQual(R.iloc[:, i], L, Q.iloc[:, j]) + iteration = 0 + while iteration < nperm: + Lperm = permuteType(L, model) + stat_perm[iteration] = QuantQual( + R.iloc[:, i], Lperm, Q.iloc[:, j] + ) + iteration += 1 + obsStat.append(stat_obs) + statType.append("F-statistic") + if test is "greater": + pval.append(np.mean(stat_perm >= stat_obs)) + if test is "lower": + pval.append(np.mean(stat_perm <= stat_obs)) + if test is "both": + pval.append(np.mean(np.abs(stat_perm) >= np.abs(stat_obs))) + self.results = DataFrame( + { + "Comparison": compName, + "Statistic": statType, + "Observed Stat": np.round(obsStat, 2), + "p-value": np.round(pval, 3), + } + ) + self.results["tail"] = test + self.nperm = nperm + self.adj = p_adjustment + + def summary(self): + if self.adj is None: + print("\nFour Corner Analysis - {0} Permutations\n".format(self.nperm)) + return self.results[ + ["Comparison", "Statistic", "Observed Stat", "tail", "p-value"] + ] + if self.adj is not None: + print( + "\nFour Corner Analysis - {0} Permutations\n{1} Correction\n".format( + self.nperm, self.adj + ) + ) + self.results["adjusted p-value"] = p_adjust( + self.results["p-value"], self.adj + ) + return self.results[ + [ + "Comparison", + "Statistic", + "Observed Stat", + "tail", + "p-value", + "adjusted p-value", + ] + ] + def QuantQuant(R, L, Q): - L2 = L.copy() - R2 = np.array(R).astype('float').flatten() - Q2 = np.array(Q).astype('float').flatten() - Ro = [] - Qo = [] - for i in range(L2.shape[0]): - for j in range(L2.shape[1]): - if L2[i,j] != 0: - Ro.extend([R2[i]]*L2[i,j]) - Qo.extend([Q2[j]]*L2[i,j]) - r = np.corrcoef(np.array(zip(Ro, Qo)), rowvar=0)[0,1] - return r + L2 = L.copy() + R2 = np.array(R).astype("float").flatten() + Q2 = np.array(Q).astype("float").flatten() + Ro = [] + Qo = [] + for i in range(L2.shape[0]): + for j in range(L2.shape[1]): + if L2[i, j] != 0: + Ro.extend([R2[i]] * L2[i, j]) + Qo.extend([Q2[j]] * L2[i, j]) + r = np.corrcoef(np.array(zip(Ro, Qo)), rowvar=0)[0, 1] + return r + def QualQual(R, L, Q, n_env, n_trait): - L2 = L.copy() - Ro = [] - Qo = [] - for i in range(L2.shape[0]): - for j in range(L2.shape[1]): - if L2[i,j] != 0: - Ro.extend([R[i]]*L2[i,j]) - Qo.extend([Q[j]]*L2[i,j]) - inflMat = np.array(zip(Ro, Qo)) - crosstab = np.zeros((n_env, n_trait)) - for i in range(n_env): - for j in range(n_trait): - crosstab[i,j] = np.argwhere((inflMat[:,0]==i) & (inflMat[:,1]==j)).shape[0] - return chisquare(crosstab.ravel())[0] + L2 = L.copy() + Ro = [] + Qo = [] + for i in range(L2.shape[0]): + for j in range(L2.shape[1]): + if L2[i, j] != 0: + Ro.extend([R[i]] * L2[i, j]) + Qo.extend([Q[j]] * L2[i, j]) + inflMat = np.array(zip(Ro, Qo)) + crosstab = np.zeros((n_env, n_trait)) + for i in range(n_env): + for j in range(n_trait): + crosstab[i, j] = np.argwhere( + (inflMat[:, 0] == i) & (inflMat[:, 1] == j) + ).shape[0] + return chisquare(crosstab.ravel())[0] + def QualQuant(R, L, Q): - L2 = L.copy() - R2 = factorize(R)[0] - Q2 = Q.copy() - Ro = [] - Qo = [] - for i in range(L2.shape[0]): - for j in range(L2.shape[1]): - if L2[i,j] != 0: - Ro.extend([R2[i]]*L2[i,j]) - Qo.extend([Q2[j]]*L2[i,j]) - inflMat = DataFrame({'Ro': Ro, 'Qo': Qo}) - btwn = inflMat.groupby('Ro')['Qo'].agg({'mean': lambda x: len(inflMat)*(x.mean() - inflMat['Qo'].mean())**2}) - btwn = btwn['mean'].sum() / (len(btwn)-1) - within = inflMat.groupby('Ro')['Qo'].agg({'mean': lambda x: np.sum((x - x.mean())**2)}) - within = within['mean'].sum() / (len(inflMat)-len(within)) - return btwn / within + L2 = L.copy() + R2 = factorize(R)[0] + Q2 = Q.copy() + Ro = [] + Qo = [] + for i in range(L2.shape[0]): + for j in range(L2.shape[1]): + if L2[i, j] != 0: + Ro.extend([R2[i]] * L2[i, j]) + Qo.extend([Q2[j]] * L2[i, j]) + inflMat = DataFrame({"Ro": Ro, "Qo": Qo}) + btwn = inflMat.groupby("Ro")["Qo"].agg( + {"mean": lambda x: len(inflMat) * (x.mean() - inflMat["Qo"].mean()) ** 2} + ) + btwn = btwn["mean"].sum() / (len(btwn) - 1) + within = inflMat.groupby("Ro")["Qo"].agg( + {"mean": lambda x: np.sum((x - x.mean()) ** 2)} + ) + within = within["mean"].sum() / (len(inflMat) - len(within)) + return btwn / within + def QuantQual(R, L, Q): - L2 = L.copy() - R2 = R.copy() - Q2 = factorize(Q)[0] - Ro = [] - Qo = [] - for i in range(L2.shape[0]): - for j in range(L2.shape[1]): - if L2[i,j] != 0: - Ro.extend([R2[i]]*L2[i,j]) - Qo.extend([Q2[j]]*L2[i,j]) - inflMat = DataFrame({'Ro': Ro, 'Qo': Qo}) - btwn = inflMat.groupby('Qo')['Ro'].agg({'mean': lambda x: len(inflMat)*(x.mean() - inflMat['Qo'].mean())**2}) - btwn = btwn['mean'].sum() / (len(btwn)-1) - within = inflMat.groupby('Qo')['Ro'].agg({'mean': lambda x: np.sum((x - x.mean())**2)}) - within = within['mean'].sum() / (len(inflMat)-len(within)) - return btwn / within + L2 = L.copy() + R2 = R.copy() + Q2 = factorize(Q)[0] + Ro = [] + Qo = [] + for i in range(L2.shape[0]): + for j in range(L2.shape[1]): + if L2[i, j] != 0: + Ro.extend([R2[i]] * L2[i, j]) + Qo.extend([Q2[j]] * L2[i, j]) + inflMat = DataFrame({"Ro": Ro, "Qo": Qo}) + btwn = inflMat.groupby("Qo")["Ro"].agg( + {"mean": lambda x: len(inflMat) * (x.mean() - inflMat["Qo"].mean()) ** 2} + ) + btwn = btwn["mean"].sum() / (len(btwn) - 1) + within = inflMat.groupby("Qo")["Ro"].agg( + {"mean": lambda x: np.sum((x - x.mean()) ** 2)} + ) + within = within["mean"].sum() / (len(inflMat) - len(within)) + return btwn / within + def permuteType(L, model): - if model==1: - Lperm = np.apply_along_axis(lambda x: np.random.permutation(x), 0, L) - if model==2: - idx = np.arange(L.shape[0]) - Lperm = L[np.random.permutation(idx),:] - if model==3: - Lperm = np.apply_along_axis(lambda x: np.random.permutation(x), 1, L) - if model==4: - idx = np.arange(L.shape[1]) - Lperm = L[:,np.random.permutation(idx)] - return Lperm + if model == 1: + Lperm = np.apply_along_axis(lambda x: np.random.permutation(x), 0, L) + if model == 2: + idx = np.arange(L.shape[0]) + Lperm = L[np.random.permutation(idx), :] + if model == 3: + Lperm = np.apply_along_axis(lambda x: np.random.permutation(x), 1, L) + if model == 4: + idx = np.arange(L.shape[1]) + Lperm = L[:, np.random.permutation(idx)] + return Lperm + def p_adjust(p, method): - if method == 'bonferroni': - return np.minimum(p*len(p), 1) - if method == 'holm': - temp = DataFrame({'p': p}) - temp.sort(columns='p', inplace=True) - temp['newID'] = range(1, len(temp)+1) - temp['p_adj'] = np.minimum(temp['p'] * (1 + len(temp) - temp['newID']), 1) - temp.sort(inplace=True) - return temp['p_adj'] - if method == 'fdr': - temp = DataFrame({'p': p}) - temp.sort(columns='p', inplace=True, ascending=False) - temp['newID'] = range(1, len(temp)+1) - temp['p_adj'] = np.minimum(1, len(temp)/temp['newID'] * temp['p']) - temp.sort(inplace=True) - return np.round(temp['p_adj'], 3) + if method == "bonferroni": + return np.minimum(p * len(p), 1) + if method == "holm": + temp = DataFrame({"p": p}) + temp.sort(columns="p", inplace=True) + temp["newID"] = range(1, len(temp) + 1) + temp["p_adj"] = np.minimum(temp["p"] * (1 + len(temp) - temp["newID"]), 1) + temp.sort(inplace=True) + return temp["p_adj"] + if method == "fdr": + temp = DataFrame({"p": p}) + temp.sort(columns="p", inplace=True, ascending=False) + temp["newID"] = range(1, len(temp) + 1) + temp["p_adj"] = np.minimum(1, len(temp) / temp["newID"] * temp["p"]) + temp.sort(inplace=True) + return np.round(temp["p_adj"], 3) diff --git a/ecopy/matrix_comp/mantel.py b/ecopy/matrix_comp/mantel.py index 8afbc53..a4a2e79 100644 --- a/ecopy/matrix_comp/mantel.py +++ b/ecopy/matrix_comp/mantel.py @@ -1,8 +1,9 @@ from pandas import DataFrame import numpy as np + class Mantel(object): - ''' + """ Docstring for function ecopy.Mantel ==================== Conducts a Mantel test for association between two square, symmetric, non-negative @@ -52,164 +53,172 @@ class Mantel(object): mant = ep.Mantel(dist1, dist2) print(mant.summary()) - ''' - def __init__(self, d1, d2, d_condition = None, test='pearson', tail='both', nperm=999): - if not isinstance(d1, (np.ndarray, DataFrame)): - msg = 'Matrix d1 must be a numpy.ndarray or pandas.DataFrame' - if not isinstance(d2, (np.ndarray, DataFrame)): - msg = 'Matrix d2 must be a numpy.ndarray or pandas.DataFrame' - if isinstance(d1, DataFrame): - d1 = np.array(d1) - if isinstance(d2, DataFrame): - d2 = np.array(d1) - if np.any(d2 < 0): - msg ='Matrix d2 cannot have negative values' - raise ValueError(msg) - if d1.shape[0] != d1.shape[1]: - msg = 'Matrix d1 must be a square, symmetric distance matrix' - raise ValueError(msg) - if d2.shape[0] != d2.shape[1]: - msg = 'Matrix d2 must be a square, symmetric distance matrix' - raise ValueError(msg) - if not np.allclose(d1.T, d1): - msg = 'Matrix d1 must be a square, symmetric distance matrix' - raise ValueError(msg) - if not np.allclose(d2.T, d2): - msg = 'Matrix d2 must be a square, symmetric distance matrix' - raise ValueError(msg) - if d1.shape[0] != d2.shape[0]: - msg = 'Matrices must have same dimensions' - raise ValueError(msg) - if test not in ['pearson', 'spearman']: - msg = 'test must be pearson or spearman' - raise ValueError(msg) - if tail not in ['greater', 'lower', 'both']: - msg = 'tail must be greater, lower, both' - raise ValueError(msg) - if nperm < 2: - msg = 'nperm must be > 2' - raise ValueError(msg) - r_perm = np.empty(nperm) - if d_condition is not None: - if d_condition.shape[0] != d_condition.shape[1]: - msg = 'Conditioning matrix must be a square, symmetric distance matrix' - raise ValueError(msg) - if not np.allclose(d_condition.T, d_condition): - msg = 'Conditioning matrix must be a square, symmetric distance matrix' - raise ValueError(msg) - resMat = residCalc(d1, d_condition) - partR = partialMantel(resMat, d2, d_condition) - self.robs = partR[0,1] - for i in range(nperm): - residMat_perm = permuteFunc(resMat) - r_star = partialMantel(residMat_perm, d2, d_condition) - r_perm[i] = r_star[0,1] - if test is 'pearson': - self.r_obs = manFunc_pears(d1, d2) - for i in range(nperm): - d1_perm = permuteFunc(d1) - r_perm[i] = manFunc_pears(d1_perm, d2) - if test is 'spearman': - self.r_obs = manFunc_spear(d1, d2) - for i in range(nperm): - d1_perm = permuteFunc(d1) - r_perm[i] = manFunc_spear(d1_perm, d2) - if tail is 'greater': - self.pval = np.mean(r_perm > self.r_obs) - elif tail is 'lower': - self.pval = np.mean(r_perm < self.r_obs) - else: - self.pval = np.mean(np.abs(r_perm) > self.r_obs) - self.test = test - self.tail = tail - self.perm = nperm - - def summary(self): - summ = '\n{0} Mantel Test\nHypothesis = {1}\n\nObserved r = {2:.3}\tp = {3:.3}\n{4} permutations'.format(self.test.title(), self.tail, self.r_obs, self.pval, self.perm) - return summ - - - -def manFunc_pears(x,y): - n = x.shape[0] - denom = n*(n-1)/2. - 1. - ui = np.triu_indices(x.shape[0]) - x2 = x.copy() - y2 = y.copy() - x2[ui] = np.nan - y2[ui] = np.nan - x_flat = x2.ravel() - y_flat = y2.ravel() - x_flat = x_flat[~np.isnan(x_flat)] - y_flat = y_flat[~np.isnan(y_flat)] - x_flat = (x_flat - x_flat.mean())/x_flat.std(ddof=1) - y_flat = (y_flat - y_flat.mean())/y_flat.std(ddof=1) - r = x_flat.dot(y_flat) / denom - return r - -def manFunc_spear(x,y): - n = x.shape[0] - denom = n*(n-1)/2. - 1. - ui = np.triu_indices(x.shape[0]) - x2 = x.copy() - y2 = y.copy() - x2[ui] = np.nan - y2[ui] = np.nan - x_flat = x2.ravel() - y_flat = y2.ravel() - x_flat = x_flat[~np.isnan(x_flat)] - y_flat = y_flat[~np.isnan(y_flat)] - x_flat = x_flat.argsort().argsort() - y_flat = y_flat.argsort().argsort() - x_flat = (x_flat - x_flat.mean())/x_flat.std(ddof=1) - y_flat = (y_flat - y_flat.mean())/y_flat.std(ddof=1) - r = x_flat.dot(y_flat) / denom - return r + """ + + def __init__( + self, d1, d2, d_condition=None, test="pearson", tail="both", nperm=999 + ): + if not isinstance(d1, (np.ndarray, DataFrame)): + msg = "Matrix d1 must be a numpy.ndarray or pandas.DataFrame" + if not isinstance(d2, (np.ndarray, DataFrame)): + msg = "Matrix d2 must be a numpy.ndarray or pandas.DataFrame" + if isinstance(d1, DataFrame): + d1 = np.array(d1) + if isinstance(d2, DataFrame): + d2 = np.array(d1) + if np.any(d2 < 0): + msg = "Matrix d2 cannot have negative values" + raise ValueError(msg) + if d1.shape[0] != d1.shape[1]: + msg = "Matrix d1 must be a square, symmetric distance matrix" + raise ValueError(msg) + if d2.shape[0] != d2.shape[1]: + msg = "Matrix d2 must be a square, symmetric distance matrix" + raise ValueError(msg) + if not np.allclose(d1.T, d1): + msg = "Matrix d1 must be a square, symmetric distance matrix" + raise ValueError(msg) + if not np.allclose(d2.T, d2): + msg = "Matrix d2 must be a square, symmetric distance matrix" + raise ValueError(msg) + if d1.shape[0] != d2.shape[0]: + msg = "Matrices must have same dimensions" + raise ValueError(msg) + if test not in ["pearson", "spearman"]: + msg = "test must be pearson or spearman" + raise ValueError(msg) + if tail not in ["greater", "lower", "both"]: + msg = "tail must be greater, lower, both" + raise ValueError(msg) + if nperm < 2: + msg = "nperm must be > 2" + raise ValueError(msg) + r_perm = np.empty(nperm) + if d_condition is not None: + if d_condition.shape[0] != d_condition.shape[1]: + msg = "Conditioning matrix must be a square, symmetric distance matrix" + raise ValueError(msg) + if not np.allclose(d_condition.T, d_condition): + msg = "Conditioning matrix must be a square, symmetric distance matrix" + raise ValueError(msg) + resMat = residCalc(d1, d_condition) + partR = partialMantel(resMat, d2, d_condition) + self.robs = partR[0, 1] + for i in range(nperm): + residMat_perm = permuteFunc(resMat) + r_star = partialMantel(residMat_perm, d2, d_condition) + r_perm[i] = r_star[0, 1] + if test is "pearson": + self.r_obs = manFunc_pears(d1, d2) + for i in range(nperm): + d1_perm = permuteFunc(d1) + r_perm[i] = manFunc_pears(d1_perm, d2) + if test is "spearman": + self.r_obs = manFunc_spear(d1, d2) + for i in range(nperm): + d1_perm = permuteFunc(d1) + r_perm[i] = manFunc_spear(d1_perm, d2) + if tail is "greater": + self.pval = np.mean(r_perm > self.r_obs) + elif tail is "lower": + self.pval = np.mean(r_perm < self.r_obs) + else: + self.pval = np.mean(np.abs(r_perm) > self.r_obs) + self.test = test + self.tail = tail + self.perm = nperm + + def summary(self): + summ = "\n{0} Mantel Test\nHypothesis = {1}\n\nObserved r = {2:.3}\tp = {3:.3}\n{4} permutations".format( + self.test.title(), self.tail, self.r_obs, self.pval, self.perm + ) + return summ + + +def manFunc_pears(x, y): + n = x.shape[0] + denom = n * (n - 1) / 2.0 - 1.0 + ui = np.triu_indices(x.shape[0]) + x2 = x.copy() + y2 = y.copy() + x2[ui] = np.nan + y2[ui] = np.nan + x_flat = x2.ravel() + y_flat = y2.ravel() + x_flat = x_flat[~np.isnan(x_flat)] + y_flat = y_flat[~np.isnan(y_flat)] + x_flat = (x_flat - x_flat.mean()) / x_flat.std(ddof=1) + y_flat = (y_flat - y_flat.mean()) / y_flat.std(ddof=1) + r = x_flat.dot(y_flat) / denom + return r + + +def manFunc_spear(x, y): + n = x.shape[0] + denom = n * (n - 1) / 2.0 - 1.0 + ui = np.triu_indices(x.shape[0]) + x2 = x.copy() + y2 = y.copy() + x2[ui] = np.nan + y2[ui] = np.nan + x_flat = x2.ravel() + y_flat = y2.ravel() + x_flat = x_flat[~np.isnan(x_flat)] + y_flat = y_flat[~np.isnan(y_flat)] + x_flat = x_flat.argsort().argsort() + y_flat = y_flat.argsort().argsort() + x_flat = (x_flat - x_flat.mean()) / x_flat.std(ddof=1) + y_flat = (y_flat - y_flat.mean()) / y_flat.std(ddof=1) + r = x_flat.dot(y_flat) / denom + return r + def permuteFunc(x): - idx = np.random.choice(x.shape[0], x.shape[0]) - xperm = x[idx,:] - for j in range(xperm.shape[0]): - xperm[j, idx[j]] = xperm[j, j] - np.fill_diagonal(xperm, 0) - return xperm + idx = np.random.choice(x.shape[0], x.shape[0]) + xperm = x[idx, :] + for j in range(xperm.shape[0]): + xperm[j, idx[j]] = xperm[j, j] + np.fill_diagonal(xperm, 0) + return xperm + def residCalc(y, z): - y2 = y.copy() - z2 = z.copy() - ui = np.triu_indices(y2.shape[0]) - iy, ix = np.indices(y2.shape) - y2[ui] = np.nan - z2[ui] = np.nan - y_flat = y2.ravel() - z_flat = z2.ravel() - ix_flat = ix.ravel() - iy_flat = iy.ravel() - ix_flat = ix_flat[~np.isnan(y_flat)] - iy_flat = iy_flat[~np.isnan(y_flat)] - y_flat = y_flat[~np.isnan(y_flat)] - z_flat = z_flat[~np.isnan(z_flat)] - Z = np.array([np.ones(len(z_flat)), z_flat]).T - params = np.linalg.lstsq(Z, y_flat)[0] - Res = y_flat - Z.dot(params) - ResMat = np.zeros(y2.shape) - for i in range(len(ix_flat)): - ResMat[ix_flat[i], iy_flat[i]] = Res[i] - for i in range(y2.shape[0]): - for j in range(y2.shape[0]): - ResMat[j,i] = ResMat[i,j] - return ResMat + y2 = y.copy() + z2 = z.copy() + ui = np.triu_indices(y2.shape[0]) + iy, ix = np.indices(y2.shape) + y2[ui] = np.nan + z2[ui] = np.nan + y_flat = y2.ravel() + z_flat = z2.ravel() + ix_flat = ix.ravel() + iy_flat = iy.ravel() + ix_flat = ix_flat[~np.isnan(y_flat)] + iy_flat = iy_flat[~np.isnan(y_flat)] + y_flat = y_flat[~np.isnan(y_flat)] + z_flat = z_flat[~np.isnan(z_flat)] + Z = np.array([np.ones(len(z_flat)), z_flat]).T + params = np.linalg.lstsq(Z, y_flat)[0] + Res = y_flat - Z.dot(params) + ResMat = np.zeros(y2.shape) + for i in range(len(ix_flat)): + ResMat[ix_flat[i], iy_flat[i]] = Res[i] + for i in range(y2.shape[0]): + for j in range(y2.shape[0]): + ResMat[j, i] = ResMat[i, j] + return ResMat + def partialMantel(x, y, z): - R = np.eye(3) - d = [x, y, z] - for i in range(3): - for j in range(3): - R[i,j] = manFunc_pears(d[i], d[j]) - R11 = R[:2, :2] - R12 = R[:2, 2].reshape(2, 1) - R21 = R12.T - Rpart = R11 - R12.dot(R21) - Dpart = np.diag(np.diag(Rpart)**-0.5) - R12_3 = Dpart.dot(Rpart).dot(Dpart) - return R12_3 + R = np.eye(3) + d = [x, y, z] + for i in range(3): + for j in range(3): + R[i, j] = manFunc_pears(d[i], d[j]) + R11 = R[:2, :2] + R12 = R[:2, 2].reshape(2, 1) + R21 = R12.T + Rpart = R11 - R12.dot(R21) + Dpart = np.diag(np.diag(Rpart) ** -0.5) + R12_3 = Dpart.dot(Rpart).dot(Dpart) + return R12_3 diff --git a/ecopy/matrix_comp/procrust_test.py b/ecopy/matrix_comp/procrust_test.py index ab64f28..999932a 100644 --- a/ecopy/matrix_comp/procrust_test.py +++ b/ecopy/matrix_comp/procrust_test.py @@ -1,8 +1,9 @@ import numpy as np from pandas import DataFrame + class procrustes_test(object): - """ + """ Docstring for function ecopy.procrustes_test ==================== Conducts permutation procrustes test of relationship @@ -41,37 +42,38 @@ class procrustes_test(object): d = ep.procrustes_test(d1, d2) print(d.summary()) """ - def __init__(self, mat1, mat2, nperm=999): - if isinstance(mat1, DataFrame): - X = np.array(mat1).astype('float') - else: - X = mat1.astype('float') - if isinstance(mat2, DataFrame): - Y = np.array(mat2).astype('float') - else: - Y = mat2.astype('float') - if X.shape[0] != Y.shape[0]: - msg = 'Matrices must have the same number of rows' - raise ValueError(msg) - X_cent = np.apply_along_axis(lambda x: x - x.mean(), 0, X) - Y_cent = np.apply_along_axis(lambda y: y - y.mean(), 0, Y) - X_cent = X_cent / np.sqrt(np.sum(X_cent**2)) - Y_cent = Y_cent / np.sqrt(np.sum(Y_cent**2)) - W = np.sum(np.linalg.svd(X_cent.T.dot(Y_cent), compute_uv=0)) - self.m12_obs = 1 - W**2 - m12_perm = np.zeros(nperm) - i = 0 - while i < nperm: - idx = np.random.permutation(range(X_cent.shape[0])) - X_perm = X_cent[idx,:] - W_perm = np.sum(np.linalg.svd(X_perm.T.dot(Y_cent), compute_uv=0)) - m12_perm[i] = 1 - W_perm**2 - i += 1 - self.pval = np.mean(m12_perm < self.m12_obs) - self.perm = nperm - - def summary(self): - summ = '\nm12 squared = {0:.3}\np = {1:.3}\npermutations = {2}'.format(self.m12_obs, self.pval, self.perm) - return summ + def __init__(self, mat1, mat2, nperm=999): + if isinstance(mat1, DataFrame): + X = np.array(mat1).astype("float") + else: + X = mat1.astype("float") + if isinstance(mat2, DataFrame): + Y = np.array(mat2).astype("float") + else: + Y = mat2.astype("float") + if X.shape[0] != Y.shape[0]: + msg = "Matrices must have the same number of rows" + raise ValueError(msg) + X_cent = np.apply_along_axis(lambda x: x - x.mean(), 0, X) + Y_cent = np.apply_along_axis(lambda y: y - y.mean(), 0, Y) + X_cent = X_cent / np.sqrt(np.sum(X_cent ** 2)) + Y_cent = Y_cent / np.sqrt(np.sum(Y_cent ** 2)) + W = np.sum(np.linalg.svd(X_cent.T.dot(Y_cent), compute_uv=0)) + self.m12_obs = 1 - W ** 2 + m12_perm = np.zeros(nperm) + i = 0 + while i < nperm: + idx = np.random.permutation(range(X_cent.shape[0])) + X_perm = X_cent[idx, :] + W_perm = np.sum(np.linalg.svd(X_perm.T.dot(Y_cent), compute_uv=0)) + m12_perm[i] = 1 - W_perm ** 2 + i += 1 + self.pval = np.mean(m12_perm < self.m12_obs) + self.perm = nperm + def summary(self): + summ = "\nm12 squared = {0:.3}\np = {1:.3}\npermutations = {2}".format( + self.m12_obs, self.pval, self.perm + ) + return summ diff --git a/ecopy/matrix_comp/rda.py b/ecopy/matrix_comp/rda.py index 708225c..a295886 100644 --- a/ecopy/matrix_comp/rda.py +++ b/ecopy/matrix_comp/rda.py @@ -2,8 +2,9 @@ from pandas import DataFrame, get_dummies import matplotlib.pyplot as plt + class rda(object): - """ + """ Docstring for function ecopy.rda ==================== Conducts RDA analysis for an site x species matrix Y and @@ -73,211 +74,282 @@ class rda(object): print(RDA.anova()) RDA.triplot() """ - def __init__(self, Y, X, scale_y=True, scale_x=False, design_x=False, varNames_y=None, varNames_x=None, rowNames=None, pTypes=None, sig=False): - tolerance = 1E-6 - if not isinstance(Y, (DataFrame, np.ndarray)): - msg = 'Matrix Y must be a pandas.DataFrame or numpy.ndarray' - raise ValueError(msg) - if not isinstance(X, (DataFrame, np.ndarray)): - msg = 'Matrix X must be a pandas.DataFrame or numpy.ndarray' - raise ValueError(msg) - if isinstance(X, DataFrame): - if X.isnull().any().any(): - msg = 'Matrix X contains null values' - raise ValueError(msg) - if isinstance(X, np.ndarray): - if X.dtype=='object': - msg = 'Matrix X cannot be a numpy.ndarray with object dtype' - raise ValueError(msg) - if np.isnan(X).any(): - msg = 'Matrix X contains null values' - raise ValueError(msg) - if isinstance(Y, DataFrame): - if Y.isnull().any().any(): - msg = 'Matrix Y contains null values' - raise ValueError(msg) - if (Y.dtypes == 'object').any(): - msg = 'Matrix Y can only contain numeric values' - raise ValueError(msg) - if isinstance(Y, np.ndarray): - if np.isnan(Y).any(): - msg = 'Matrix Y contains null values' - raise ValueError(msg) - if varNames_y is None: - if isinstance(Y, DataFrame): - varNames_y = Y.columns - elif isinstance(Y, np.ndarray): - varNames_y = ['Sp {0}'.format(x) for x in range(1, Y.shape[1]+1)] - if varNames_x is None: - if isinstance(X, DataFrame): - varNames_x = X.columns - elif isinstance(X, np.ndarray): - varNames_x = ['Pred {0}'.format(x) for x in range(1, X.shape[1]+1)] - if rowNames is None: - if isinstance(Y, DataFrame): - rowNames = Y.index.values - elif isinstance(Y, np.ndarray): - rowNames = ['Site {0}'.format(x) for x in range(1, Y.shape[0]+1)] - self.y_mat = np.array(Y, dtype='float') - if design_x: - self.x_mat = np.array(X, dtype='float') - if pTypes is None: - pTypes = ['q']*self.x_mat.shape[1] - if not design_x and isinstance(X, DataFrame): - self.x_mat, varNames_x, pTypes = dummyMat(X, scale_x) - elif not design_x and isinstance(X, np.ndarray): - self.x_mat = np.array(X, dtype='float') - if pTypes is None: - pTypes = ['q']*self.x_mat.shape[1] - msg = 'Warning: X is a numpy array but not a design matrix. Make sure that matrix X represents the model you wish to analyze. Use patsy.dmatrix if unsure' - print(msg) - if not set(pTypes).issubset(['f', 'q']): - msg = 'pTypes must contain only f or q characters' - raise ValueError(msg) - self.y_mat = np.apply_along_axis(lambda x: x-x.mean(), 0, self.y_mat) - if scale_y: - self.y_mat = np.apply_along_axis(lambda x: x /x.std(ddof=1), 0, self.y_mat) - B = np.linalg.pinv(self.x_mat.T.dot(self.x_mat)).dot(self.x_mat.T).dot(self.y_mat) - yhat = self.x_mat.dot(B) - yhat_cov = np.cov(yhat, rowvar=0) - evals, evecs = np.linalg.eig(yhat_cov) - evals = np.real(evals) - idx = evals.argsort()[::-1] - evals = evals[idx] - evecs = evecs[:,idx] - RDA_evals = evals[evals>tolerance] - U = np.real(evecs[:,evals>tolerance]) - F = self.y_mat.dot(U) - Z = yhat.dot(U) - C = B.dot(U) - self.spScores = DataFrame(U.dot(np.diag(RDA_evals**0.5)), index=varNames_y) - self.linSites = DataFrame(Z.dot(np.diag(RDA_evals**-0.5)), index=rowNames) - self.siteScores = DataFrame(F.dot(np.diag(RDA_evals**-0.5)), index=rowNames) - self.predScores = DataFrame(C, index=varNames_x) - RDA_names = ['RDA Axis {0}'.format(x) for x in range(1, len(RDA_evals)+1)] - self.spScores.columns=RDA_names - self.linSites.columns=RDA_names - self.siteScores.columns=RDA_names - self.predScores.columns=RDA_names - self.RDA_evals = RDA_evals - self.pTypes = pTypes - self.corr = np.zeros((self.x_mat.shape[1], len(RDA_evals))) - for i in range(self.x_mat.shape[1]): - for j in range(len(RDA_evals)): - self.corr[i,j] = np.corrcoef(self.x_mat[:,i], self.linSites.iloc[:,j])[0,1] - self.corr = DataFrame(self.corr, index=varNames_x) - self.corr.columns = RDA_names - SSY = np.sum(self.y_mat**2) - SSYhat = np.sum(yhat**2) - self.R2 = SSYhat / SSY - self.n = self.y_mat.shape[0] - self.m = len(B) - self.R2a = 1. - (1. - self.R2)*((self.n-1.)/(self.n-self.m-1.)) - - residuals = yhat - self.y_mat - res_cov = np.cov(residuals, rowvar=0) - res_evals, res_evecs = np.linalg.eig(res_cov) - res_evals = np.real(res_evals[res_evals.argsort()[::-1]]) - res_evecs = np.real(res_evecs[:,res_evals.argsort()[::-1]]) - self.resid_evals = res_evals[res_evals > tolerance] - residU = res_evecs[:,res_evals > tolerance] - self.resid_spScores = DataFrame(residU.dot(np.diag(self.resid_evals**0.5)), index=varNames_y) - self.resid_siteScores = DataFrame(residuals.dot(residU).dot(np.diag(self.resid_evals**0.5)), index=rowNames) - PC_names = ['PC Axis {0}'.format(x) for x in range(1, len(self.resid_evals)+1)] - self.resid_spScores.columns = PC_names - self.resid_siteScores.columns = PC_names - totevals = np.append(self.RDA_evals, self.resid_evals) - sds = np.sqrt(totevals) - props = totevals / totevals.sum() - cums = np.cumsum(totevals) / totevals.sum() - RDA_names.extend(PC_names) - self.imp = DataFrame(np.vstack((sds, props, cums)), index = ['Std Dev', 'Prop Var', 'Cum Var']) - self.imp.columns = RDA_names + def __init__( + self, + Y, + X, + scale_y=True, + scale_x=False, + design_x=False, + varNames_y=None, + varNames_x=None, + rowNames=None, + pTypes=None, + sig=False, + ): + tolerance = 1e-6 + if not isinstance(Y, (DataFrame, np.ndarray)): + msg = "Matrix Y must be a pandas.DataFrame or numpy.ndarray" + raise ValueError(msg) + if not isinstance(X, (DataFrame, np.ndarray)): + msg = "Matrix X must be a pandas.DataFrame or numpy.ndarray" + raise ValueError(msg) + if isinstance(X, DataFrame): + if X.isnull().any().any(): + msg = "Matrix X contains null values" + raise ValueError(msg) + if isinstance(X, np.ndarray): + if X.dtype == "object": + msg = "Matrix X cannot be a numpy.ndarray with object dtype" + raise ValueError(msg) + if np.isnan(X).any(): + msg = "Matrix X contains null values" + raise ValueError(msg) + if isinstance(Y, DataFrame): + if Y.isnull().any().any(): + msg = "Matrix Y contains null values" + raise ValueError(msg) + if (Y.dtypes == "object").any(): + msg = "Matrix Y can only contain numeric values" + raise ValueError(msg) + if isinstance(Y, np.ndarray): + if np.isnan(Y).any(): + msg = "Matrix Y contains null values" + raise ValueError(msg) + if varNames_y is None: + if isinstance(Y, DataFrame): + varNames_y = Y.columns + elif isinstance(Y, np.ndarray): + varNames_y = ["Sp {0}".format(x) for x in range(1, Y.shape[1] + 1)] + if varNames_x is None: + if isinstance(X, DataFrame): + varNames_x = X.columns + elif isinstance(X, np.ndarray): + varNames_x = ["Pred {0}".format(x) for x in range(1, X.shape[1] + 1)] + if rowNames is None: + if isinstance(Y, DataFrame): + rowNames = Y.index.values + elif isinstance(Y, np.ndarray): + rowNames = ["Site {0}".format(x) for x in range(1, Y.shape[0] + 1)] + self.y_mat = np.array(Y, dtype="float") + if design_x: + self.x_mat = np.array(X, dtype="float") + if pTypes is None: + pTypes = ["q"] * self.x_mat.shape[1] + if not design_x and isinstance(X, DataFrame): + self.x_mat, varNames_x, pTypes = dummyMat(X, scale_x) + elif not design_x and isinstance(X, np.ndarray): + self.x_mat = np.array(X, dtype="float") + if pTypes is None: + pTypes = ["q"] * self.x_mat.shape[1] + msg = "Warning: X is a numpy array but not a design matrix. Make sure that matrix X represents the model you wish to analyze. Use patsy.dmatrix if unsure" + print(msg) + if not set(pTypes).issubset(["f", "q"]): + msg = "pTypes must contain only f or q characters" + raise ValueError(msg) + self.y_mat = np.apply_along_axis(lambda x: x - x.mean(), 0, self.y_mat) + if scale_y: + self.y_mat = np.apply_along_axis(lambda x: x / x.std(ddof=1), 0, self.y_mat) + B = ( + np.linalg.pinv(self.x_mat.T.dot(self.x_mat)) + .dot(self.x_mat.T) + .dot(self.y_mat) + ) + yhat = self.x_mat.dot(B) + yhat_cov = np.cov(yhat, rowvar=0) + evals, evecs = np.linalg.eig(yhat_cov) + evals = np.real(evals) + idx = evals.argsort()[::-1] + evals = evals[idx] + evecs = evecs[:, idx] + RDA_evals = evals[evals > tolerance] + U = np.real(evecs[:, evals > tolerance]) + F = self.y_mat.dot(U) + Z = yhat.dot(U) + C = B.dot(U) + self.spScores = DataFrame(U.dot(np.diag(RDA_evals ** 0.5)), index=varNames_y) + self.linSites = DataFrame(Z.dot(np.diag(RDA_evals ** -0.5)), index=rowNames) + self.siteScores = DataFrame(F.dot(np.diag(RDA_evals ** -0.5)), index=rowNames) + self.predScores = DataFrame(C, index=varNames_x) + RDA_names = ["RDA Axis {0}".format(x) for x in range(1, len(RDA_evals) + 1)] + self.spScores.columns = RDA_names + self.linSites.columns = RDA_names + self.siteScores.columns = RDA_names + self.predScores.columns = RDA_names + self.RDA_evals = RDA_evals + self.pTypes = pTypes + self.corr = np.zeros((self.x_mat.shape[1], len(RDA_evals))) + for i in range(self.x_mat.shape[1]): + for j in range(len(RDA_evals)): + self.corr[i, j] = np.corrcoef( + self.x_mat[:, i], self.linSites.iloc[:, j] + )[0, 1] + self.corr = DataFrame(self.corr, index=varNames_x) + self.corr.columns = RDA_names + SSY = np.sum(self.y_mat ** 2) + SSYhat = np.sum(yhat ** 2) + self.R2 = SSYhat / SSY + self.n = self.y_mat.shape[0] + self.m = len(B) + self.R2a = 1.0 - (1.0 - self.R2) * ((self.n - 1.0) / (self.n - self.m - 1.0)) + + residuals = yhat - self.y_mat + res_cov = np.cov(residuals, rowvar=0) + res_evals, res_evecs = np.linalg.eig(res_cov) + res_evals = np.real(res_evals[res_evals.argsort()[::-1]]) + res_evecs = np.real(res_evecs[:, res_evals.argsort()[::-1]]) + self.resid_evals = res_evals[res_evals > tolerance] + residU = res_evecs[:, res_evals > tolerance] + self.resid_spScores = DataFrame( + residU.dot(np.diag(self.resid_evals ** 0.5)), index=varNames_y + ) + self.resid_siteScores = DataFrame( + residuals.dot(residU).dot(np.diag(self.resid_evals ** 0.5)), index=rowNames + ) + PC_names = [ + "PC Axis {0}".format(x) for x in range(1, len(self.resid_evals) + 1) + ] + self.resid_spScores.columns = PC_names + self.resid_siteScores.columns = PC_names + totevals = np.append(self.RDA_evals, self.resid_evals) + sds = np.sqrt(totevals) + props = totevals / totevals.sum() + cums = np.cumsum(totevals) / totevals.sum() + RDA_names.extend(PC_names) + self.imp = DataFrame( + np.vstack((sds, props, cums)), index=["Std Dev", "Prop Var", "Cum Var"] + ) + self.imp.columns = RDA_names + def summary(self, n=10): + print( + "\nTotal Variance = {0:.3}".format( + np.sum(self.RDA_evals) + np.sum(self.resid_evals) + ) + ) + print("Constrained Variance = {0:.3}".format(np.sum(self.RDA_evals))) + print("Residual Variance = {0:.3}".format(np.sum(self.resid_evals))) + print("R2 = {0:.3}".format(self.R2)) + print("Adjusted R2 = {0:.3}".format(self.R2a)) - def summary(self, n=10): - print('\nTotal Variance = {0:.3}'.format(np.sum(self.RDA_evals) + np.sum(self.resid_evals))) - print('Constrained Variance = {0:.3}'.format(np.sum(self.RDA_evals))) - print('Residual Variance = {0:.3}'.format(np.sum(self.resid_evals))) - print('R2 = {0:.3}'.format(self.R2)) - print('Adjusted R2 = {0:.3}'.format(self.R2a)) + def anova(self, nperm=999): + constrained = np.sum(self.RDA_evals) + resid = np.sum(self.resid_evals) + Fobs = (constrained / len(self.RDA_evals)) / (resid / len(self.resid_evals)) + Fperm = np.empty(nperm) + for i in range(nperm): + idx = np.random.choice(self.n, self.n) + y_perm = self.y_mat[idx, :] + B = ( + np.linalg.pinv(self.x_mat.T.dot(self.x_mat)) + .dot(self.x_mat.T) + .dot(y_perm) + ) + yhat_perm = self.x_mat.dot(B) + cov_perm = np.cov(yhat_perm, rowvar=0) + evals_perm, evecs_perm = np.linalg.eig(cov_perm) + evals_perm = np.real(evals_perm) + evals_perm = evals_perm[evals_perm.argsort()[::-1]] + evals_perm = evals_perm[evals_perm > 1e-6] + residuals = yhat_perm - self.y_mat + res_cov = np.cov(residuals, rowvar=0) + res_evals, res_evecs = np.linalg.eig(res_cov) + res_evals = np.real(res_evals[res_evals.argsort()[::-1]]) + res_evals[res_evals > 1e-6] + constrained = np.sum(evals_perm) + resid = np.sum(res_evals) + Fperm[i] = (constrained / len(evals_perm)) / (resid / len(res_evals)) + print("Model F-statistic = {0:.3}".format(Fobs)) + print("p = {0:.4}".format(np.mean(Fperm > Fobs))) - def anova(self, nperm=999): - constrained = np.sum(self.RDA_evals) - resid = np.sum(self.resid_evals) - Fobs = (constrained/len(self.RDA_evals)) / (resid/len(self.resid_evals)) - Fperm = np.empty(nperm) - for i in range(nperm): - idx = np.random.choice(self.n, self.n) - y_perm = self.y_mat[idx,:] - B = np.linalg.pinv(self.x_mat.T.dot(self.x_mat)).dot(self.x_mat.T).dot(y_perm) - yhat_perm = self.x_mat.dot(B) - cov_perm = np.cov(yhat_perm, rowvar=0) - evals_perm, evecs_perm = np.linalg.eig(cov_perm) - evals_perm = np.real(evals_perm) - evals_perm = evals_perm[evals_perm.argsort()[::-1]] - evals_perm = evals_perm[evals_perm>1E-6] - residuals = yhat_perm - self.y_mat - res_cov = np.cov(residuals, rowvar=0) - res_evals, res_evecs = np.linalg.eig(res_cov) - res_evals = np.real(res_evals[res_evals.argsort()[::-1]]) - res_evals[res_evals > 1E-6] - constrained = np.sum(evals_perm) - resid = np.sum(res_evals) - Fperm[i] = (constrained / len(evals_perm)) / (resid/len(res_evals)) - print('Model F-statistic = {0:.3}'.format(Fobs)) - print('p = {0:.4}'.format(np.mean(Fperm > Fobs))) + def triplot(self, xax=1, yax=2): + xplot = xax - 1 + yplot = yax - 1 + f, ax = plt.subplots() + for i in range(self.spScores.shape[0]): + ax.plot(self.spScores.iloc[i, xplot], self.spScores.iloc[i, yplot], ms=0) + ax.text( + self.spScores.iloc[i, xplot], + self.spScores.iloc[i, yplot], + self.spScores.index.values[i], + color="r", + ha="center", + va="center", + ) + for i in range(self.siteScores.shape[0]): + ax.plot( + self.siteScores.iloc[i, xplot], self.siteScores.iloc[i, yplot], ms=0 + ) + ax.text( + self.siteScores.iloc[i, xplot], + self.siteScores.iloc[i, yplot], + self.siteScores.index.values[i], + color="k", + ha="center", + va="center", + ) + for i in range(self.corr.shape[0]): + if self.pTypes[i] == "q": + ax.arrow( + 0, + 0, + self.corr.iloc[i, xplot], + self.corr.iloc[i, yplot], + color="b", + head_width=0.1, + ) + ax.text( + self.corr.iloc[i, xplot] * 1.2, + self.corr.iloc[i, yplot] * 1.2, + self.corr.index.values[i], + color="b", + ha="center", + va="center", + ) + elif self.pTypes[i] == "f": + ax.text( + self.corr.iloc[i, xplot], + self.corr.iloc[i, yplot], + self.corr.index.values[i], + color="b", + ha="center", + va="bottom", + ) + ax.set_xlabel("RDA {0}".format(xax)) + ax.set_ylabel("RDA {0}".format(yax)) + plt.show() - def triplot(self, xax=1, yax=2): - xplot = xax-1 - yplot = yax-1 - f, ax = plt.subplots() - for i in range(self.spScores.shape[0]): - ax.plot(self.spScores.iloc[i,xplot], self.spScores.iloc[i,yplot], ms=0) - ax.text(self.spScores.iloc[i,xplot], self.spScores.iloc[i,yplot], self.spScores.index.values[i], color='r', ha='center', va='center') - for i in range(self.siteScores.shape[0]): - ax.plot(self.siteScores.iloc[i,xplot], self.siteScores.iloc[i,yplot], ms=0) - ax.text(self.siteScores.iloc[i,xplot], self.siteScores.iloc[i,yplot], self.siteScores.index.values[i], color='k', ha='center', va='center') - for i in range(self.corr.shape[0]): - if self.pTypes[i] == 'q': - ax.arrow(0, 0, self.corr.iloc[i,xplot], self.corr.iloc[i,yplot], color='b', head_width=0.1) - ax.text(self.corr.iloc[i,xplot]*1.2, self.corr.iloc[i,yplot]*1.2, self.corr.index.values[i], color='b', ha='center', va='center') - elif self.pTypes[i] == 'f': - ax.text(self.corr.iloc[i,xplot], self.corr.iloc[i,yplot], self.corr.index.values[i], color='b', ha='center', va='bottom') - ax.set_xlabel('RDA {0}'.format(xax)) - ax.set_ylabel('RDA {0}'.format(yax)) - plt.show() def dummyMat(matrix, scale): - rows = matrix.shape[0] - columns = matrix.shape[1] - columnType = np.array([""]*columns) - for col in range(columns): - id1 = 'q' - if matrix.iloc[:,col].dtype=='int': - matrix.iloc[:,col] = matrix.iloc[:,col].apply(float) - elif matrix.iloc[:,col].dtype=='object': - id1 = 'f' - columnType[col] = id1 - modMat = np.array([0]*rows).reshape(rows, 1) - modNames = ['0'] - pType = ['0'] - for col in range(columns): - if columnType[col]=='q': - w = matrix.iloc[:,col] - w = (w - np.mean(w)) - if scale: - w = w / np.std(w, ddof=1) - modMat = np.hstack((modMat, w.reshape(rows, 1))) - modNames.append(matrix.columns[col]) - pType.append('q') - elif columnType[col]=='f': - colName = matrix.columns[col] - w = get_dummies(matrix.iloc[:,col]) - levels = [colName + ':' + str(x) for x in w.columns] - modMat = np.hstack((modMat, w)) - modNames.extend(levels) - pType.extend(['f']*len(levels)) - return modMat[:,1:], modNames[1:], pType[1:] + rows = matrix.shape[0] + columns = matrix.shape[1] + columnType = np.array([""] * columns) + for col in range(columns): + id1 = "q" + if matrix.iloc[:, col].dtype == "int": + matrix.iloc[:, col] = matrix.iloc[:, col].apply(float) + elif matrix.iloc[:, col].dtype == "object": + id1 = "f" + columnType[col] = id1 + modMat = np.array([0] * rows).reshape(rows, 1) + modNames = ["0"] + pType = ["0"] + for col in range(columns): + if columnType[col] == "q": + w = matrix.iloc[:, col] + w = w - np.mean(w) + if scale: + w = w / np.std(w, ddof=1) + modMat = np.hstack((modMat, w.reshape(rows, 1))) + modNames.append(matrix.columns[col]) + pType.append("q") + elif columnType[col] == "f": + colName = matrix.columns[col] + w = get_dummies(matrix.iloc[:, col]) + levels = [colName + ":" + str(x) for x in w.columns] + modMat = np.hstack((modMat, w)) + modNames.extend(levels) + pType.extend(["f"] * len(levels)) + return modMat[:, 1:], modNames[1:], pType[1:] diff --git a/ecopy/matrix_comp/rlq.py b/ecopy/matrix_comp/rlq.py index 1f7a257..a253068 100644 --- a/ecopy/matrix_comp/rlq.py +++ b/ecopy/matrix_comp/rlq.py @@ -3,8 +3,9 @@ import matplotlib.pyplot as plt from ..base_funcs import wt_scale + class rlq(object): - """ + """ Docstring for function ecopy.rlq ==================== Conducts RLQ analysis for an environmental matrix (R), @@ -56,163 +57,242 @@ class rlq(object): print(rlq_ouput.summary().iloc[:,:3]) rlq_output.biplot() """ - def __init__(self, R, L, Q, ndim=2): - if not isinstance(R,DataFrame): - msg = 'Matrix R must be a pandas.DataFrame' - raise ValueError(msg) - if not isinstance(L, DataFrame): - msg = 'Matrix L must be a pandas.DataFrame' - raise ValueError(msg) - if L.dtypes.any()=='object': - msg ='Matrix L must be only numeric' - raise ValueError(msg) - if not isinstance(Q, DataFrame): - msg = 'Matrix Q must be a pandas.DataFrame' - - nr_sp = L.shape[0] - nc_sp = L.shape[1] - nr_e = R.shape[0] - nc_e = R.shape[1] - nr_t = Q.shape[0] - nc_t = Q.shape[1] - - if nr_e != nr_sp: - msg = 'Matrices R and L must have the same number of rows' - raise ValueError(msg) - if nr_t != nc_sp: - msg = 'The number of matrix L columns must equal the number of matrix Q rows' - if nr_e < nc_e: - msg = 'Number of columns cannot exceed number of rows in environment matrix' - raise ValueError(msg) - if nr_t < nc_t: - msg = 'Number of columns cannot exceed number of rows in trait matrix' - raise ValueError(msg) - - Lmat = np.array(L, dtype='float') - if Lmat.sum(axis=0).any()==0: - msg = 'Matrix L has at least one empty column' - raise ValueError(msg) - if Lmat.sum(axis=1).any()==0: - msg = 'Matrix L has at least one empty row' - raise ValueError(msg) - - Lmat = Lmat / Lmat.sum() - row_w = Lmat.sum(axis=1) - col_w = Lmat.sum(axis=0) - Lmat = np.apply_along_axis(lambda x: x / row_w, 0, Lmat) - Lmat = np.apply_along_axis(lambda x: x/col_w, 1, Lmat) - 1 - envMat, envNames, envWeights = dummyMat(R, nr_e, nc_e, row_w) - traitMat, traitNames, traitWeights = dummyMat(Q, nr_t, nc_t, col_w) - RLQmat = envMat.T.dot(np.diag(row_w)).dot(Lmat).dot(np.diag(col_w)).dot(traitMat) - RLQmat = DataFrame(RLQmat, index=envNames) - RLQmat.columns = traitNames - axes, rowcoords, colcoords, components, self.evals = ordfunc(RLQmat, traitWeights, envWeights, ndim) - self.evals = np.real(self.evals) - self.traitVecs = np.apply_along_axis(lambda x: x*traitWeights, 0, axes) - traitScores = DataFrame(np.real(traitMat.dot(self.traitVecs)), index=L.columns) - traitScores.columns = ['Trait Axis {0}'.format(x) for x in range(1,ndim+1)] - self.traitVecs = DataFrame(np.real(axes), index=traitNames) - self.traitVecs.columns = ['Trait Vector {0}'.format(x) for x in range(1,ndim+1)] - self.normedTraits = normalize(traitScores, col_w) - self.envVecs = np.apply_along_axis(lambda x: x*envWeights, 0, components) - envScores = DataFrame(np.real(envMat.dot(self.envVecs)), index=L.index) - envScores.columns = ['Environment Component {0}'.format(x) for x in range(1,ndim+1)] - self.normedEnv = normalize(envScores, row_w) - self.envVecs = DataFrame(np.real(components), index=envNames) - self.envVecs.columns = ['Environmental Vector {0}'.format(x) for x in range(1,ndim+1)] - - def summary(self): - axis_names = ['Axis {0}'.format(str(i+1)) for i in range(len(self.evals))] - sds = np.sqrt(self.evals) - props = self.evals / self.evals.sum() - cumprop = np.cumsum(self.evals) / self.evals.sum() - summDF = DataFrame(np.vstack((sds, props, cumprop)), index=['Std. Dev', 'Prop Var', 'Cum Var']) - summDF.columns = axis_names - return summDF - - def biplot(self, xax=1, yax=2): - f, ax = plt.subplots(2, 2, figsize=(12, 12)) - ax[0,0].plot(self.normedTraits.iloc[:,xax-1], self.normedTraits.iloc[:,yax-1],'o', ms=0) - for i in range(self.normedTraits.shape[0]): - ax[0,0].text(self.normedTraits.iloc[i, xax-1], self.normedTraits.iloc[i, yax-1], self.normedTraits.index.values[i], ha='center', va='center') - ax[0,0].set_xlabel(self.normedTraits.columns[0]) - ax[0,0].set_ylabel(self.normedTraits.columns[1]) - ax[0,1].plot(self.normedEnv.iloc[:,xax-1], self.normedEnv.iloc[:,yax-1],'o', ms=0) - for i in range(self.normedEnv.shape[0]): - ax[0,1].text(self.normedEnv.iloc[i, xax-1], self.normedEnv.iloc[i, yax-1], self.normedEnv.index.values[i], ha='center', va='center') - ax[0,1].set_xlabel(self.normedEnv.columns[0]) - ax[0,1].set_ylabel(self.normedEnv.columns[1]) - for i in range(self.traitVecs.shape[0]): - xlim = [np.min(self.traitVecs.iloc[:,xax-1])*1.1, np.max(self.traitVecs.iloc[:,xax-1])*1.1] - ylim = [np.min(self.traitVecs.iloc[:,yax-1])*1.1, np.max(self.traitVecs.iloc[:,yax-1])*1.1] - ax[1,0].arrow(0, 0, self.traitVecs.iloc[i,xax-1], self.traitVecs.iloc[i,yax-1], color='red', head_width=np.ptp(xlim)*0.01) - ax[1,0].text(self.traitVecs.iloc[i,xax-1]*1.1, self.traitVecs.iloc[i,yax-1]*1.1, self.traitVecs.index.values[i], color='r', ha='center', va='center') - ax[1,0].set_xlim(xlim) - ax[1,0].set_ylim(ylim) - ax[1,0].set_xlabel(self.traitVecs.columns[0]) - ax[1,0].set_ylabel(self.traitVecs.columns[1]) - for i in range(self.envVecs.shape[0]): - xlim = [np.min(self.envVecs.iloc[:,xax-1])*1.1, np.max(self.envVecs.iloc[:,xax-1])*1.1] - ylim = [np.min(self.envVecs.iloc[:,yax-1])*1.1, np.max(self.envVecs.iloc[:,yax-1])*1.1] - ax[1,1].arrow(0, 0, self.envVecs.iloc[i,xax-1], self.envVecs.iloc[i,yax-1], color='blue', head_width=np.ptp(xlim)*0.01) - ax[1,1].text(self.envVecs.iloc[i,xax-1]*1.1, self.envVecs.iloc[i,yax-1]*1.1, self.envVecs.index.values[i], color='blue', ha='center', va='center') - ax[1,1].set_xlim(xlim) - ax[1,1].set_ylim(ylim) - ax[1,1].set_xlabel(self.envVecs.columns[0]) - ax[1,1].set_ylabel(self.envVecs.columns[1]) - plt.subplots_adjust(wspace=0.3, hspace=0.3) - plt.show() + + def __init__(self, R, L, Q, ndim=2): + if not isinstance(R, DataFrame): + msg = "Matrix R must be a pandas.DataFrame" + raise ValueError(msg) + if not isinstance(L, DataFrame): + msg = "Matrix L must be a pandas.DataFrame" + raise ValueError(msg) + if L.dtypes.any() == "object": + msg = "Matrix L must be only numeric" + raise ValueError(msg) + if not isinstance(Q, DataFrame): + msg = "Matrix Q must be a pandas.DataFrame" + + nr_sp = L.shape[0] + nc_sp = L.shape[1] + nr_e = R.shape[0] + nc_e = R.shape[1] + nr_t = Q.shape[0] + nc_t = Q.shape[1] + + if nr_e != nr_sp: + msg = "Matrices R and L must have the same number of rows" + raise ValueError(msg) + if nr_t != nc_sp: + msg = ( + "The number of matrix L columns must equal the number of matrix Q rows" + ) + if nr_e < nc_e: + msg = "Number of columns cannot exceed number of rows in environment matrix" + raise ValueError(msg) + if nr_t < nc_t: + msg = "Number of columns cannot exceed number of rows in trait matrix" + raise ValueError(msg) + + Lmat = np.array(L, dtype="float") + if Lmat.sum(axis=0).any() == 0: + msg = "Matrix L has at least one empty column" + raise ValueError(msg) + if Lmat.sum(axis=1).any() == 0: + msg = "Matrix L has at least one empty row" + raise ValueError(msg) + + Lmat = Lmat / Lmat.sum() + row_w = Lmat.sum(axis=1) + col_w = Lmat.sum(axis=0) + Lmat = np.apply_along_axis(lambda x: x / row_w, 0, Lmat) + Lmat = np.apply_along_axis(lambda x: x / col_w, 1, Lmat) - 1 + envMat, envNames, envWeights = dummyMat(R, nr_e, nc_e, row_w) + traitMat, traitNames, traitWeights = dummyMat(Q, nr_t, nc_t, col_w) + RLQmat = ( + envMat.T.dot(np.diag(row_w)).dot(Lmat).dot(np.diag(col_w)).dot(traitMat) + ) + RLQmat = DataFrame(RLQmat, index=envNames) + RLQmat.columns = traitNames + axes, rowcoords, colcoords, components, self.evals = ordfunc( + RLQmat, traitWeights, envWeights, ndim + ) + self.evals = np.real(self.evals) + self.traitVecs = np.apply_along_axis(lambda x: x * traitWeights, 0, axes) + traitScores = DataFrame(np.real(traitMat.dot(self.traitVecs)), index=L.columns) + traitScores.columns = ["Trait Axis {0}".format(x) for x in range(1, ndim + 1)] + self.traitVecs = DataFrame(np.real(axes), index=traitNames) + self.traitVecs.columns = [ + "Trait Vector {0}".format(x) for x in range(1, ndim + 1) + ] + self.normedTraits = normalize(traitScores, col_w) + self.envVecs = np.apply_along_axis(lambda x: x * envWeights, 0, components) + envScores = DataFrame(np.real(envMat.dot(self.envVecs)), index=L.index) + envScores.columns = [ + "Environment Component {0}".format(x) for x in range(1, ndim + 1) + ] + self.normedEnv = normalize(envScores, row_w) + self.envVecs = DataFrame(np.real(components), index=envNames) + self.envVecs.columns = [ + "Environmental Vector {0}".format(x) for x in range(1, ndim + 1) + ] + + def summary(self): + axis_names = ["Axis {0}".format(str(i + 1)) for i in range(len(self.evals))] + sds = np.sqrt(self.evals) + props = self.evals / self.evals.sum() + cumprop = np.cumsum(self.evals) / self.evals.sum() + summDF = DataFrame( + np.vstack((sds, props, cumprop)), index=["Std. Dev", "Prop Var", "Cum Var"] + ) + summDF.columns = axis_names + return summDF + + def biplot(self, xax=1, yax=2): + f, ax = plt.subplots(2, 2, figsize=(12, 12)) + ax[0, 0].plot( + self.normedTraits.iloc[:, xax - 1], + self.normedTraits.iloc[:, yax - 1], + "o", + ms=0, + ) + for i in range(self.normedTraits.shape[0]): + ax[0, 0].text( + self.normedTraits.iloc[i, xax - 1], + self.normedTraits.iloc[i, yax - 1], + self.normedTraits.index.values[i], + ha="center", + va="center", + ) + ax[0, 0].set_xlabel(self.normedTraits.columns[0]) + ax[0, 0].set_ylabel(self.normedTraits.columns[1]) + ax[0, 1].plot( + self.normedEnv.iloc[:, xax - 1], self.normedEnv.iloc[:, yax - 1], "o", ms=0 + ) + for i in range(self.normedEnv.shape[0]): + ax[0, 1].text( + self.normedEnv.iloc[i, xax - 1], + self.normedEnv.iloc[i, yax - 1], + self.normedEnv.index.values[i], + ha="center", + va="center", + ) + ax[0, 1].set_xlabel(self.normedEnv.columns[0]) + ax[0, 1].set_ylabel(self.normedEnv.columns[1]) + for i in range(self.traitVecs.shape[0]): + xlim = [ + np.min(self.traitVecs.iloc[:, xax - 1]) * 1.1, + np.max(self.traitVecs.iloc[:, xax - 1]) * 1.1, + ] + ylim = [ + np.min(self.traitVecs.iloc[:, yax - 1]) * 1.1, + np.max(self.traitVecs.iloc[:, yax - 1]) * 1.1, + ] + ax[1, 0].arrow( + 0, + 0, + self.traitVecs.iloc[i, xax - 1], + self.traitVecs.iloc[i, yax - 1], + color="red", + head_width=np.ptp(xlim) * 0.01, + ) + ax[1, 0].text( + self.traitVecs.iloc[i, xax - 1] * 1.1, + self.traitVecs.iloc[i, yax - 1] * 1.1, + self.traitVecs.index.values[i], + color="r", + ha="center", + va="center", + ) + ax[1, 0].set_xlim(xlim) + ax[1, 0].set_ylim(ylim) + ax[1, 0].set_xlabel(self.traitVecs.columns[0]) + ax[1, 0].set_ylabel(self.traitVecs.columns[1]) + for i in range(self.envVecs.shape[0]): + xlim = [ + np.min(self.envVecs.iloc[:, xax - 1]) * 1.1, + np.max(self.envVecs.iloc[:, xax - 1]) * 1.1, + ] + ylim = [ + np.min(self.envVecs.iloc[:, yax - 1]) * 1.1, + np.max(self.envVecs.iloc[:, yax - 1]) * 1.1, + ] + ax[1, 1].arrow( + 0, + 0, + self.envVecs.iloc[i, xax - 1], + self.envVecs.iloc[i, yax - 1], + color="blue", + head_width=np.ptp(xlim) * 0.01, + ) + ax[1, 1].text( + self.envVecs.iloc[i, xax - 1] * 1.1, + self.envVecs.iloc[i, yax - 1] * 1.1, + self.envVecs.index.values[i], + color="blue", + ha="center", + va="center", + ) + ax[1, 1].set_xlim(xlim) + ax[1, 1].set_ylim(ylim) + ax[1, 1].set_xlabel(self.envVecs.columns[0]) + ax[1, 1].set_ylabel(self.envVecs.columns[1]) + plt.subplots_adjust(wspace=0.3, hspace=0.3) + plt.show() + def dummyMat(matrix, rows, columns, row_w): - columnType = np.array([""]*columns) - weights = np.array(np.nan) - for col in range(columns): - id1 = 'q' - if matrix.iloc[:,col].dtype=='int': - matrix.iloc[:,col] = matrix.iloc[:,col].apply(float) - elif matrix.iloc[:,col].dtype=='object': - id1 = 'f' - columnType[col] = id1 - modMat = np.array([0]*rows).reshape(rows, 1) - modNames = ['0'] - for col in range(columns): - if columnType[col]=='q': - modMat = np.hstack((modMat, wt_scale(matrix.iloc[:,col], row_w, bias=1).reshape(rows, 1))) - modNames.append(matrix.columns[col]) - weights = np.append(weights, 1) - elif columnType[col]=='f': - colName = matrix.columns[col] - w = get_dummies(matrix.iloc[:,col]) - levels = [colName + ':' + x for x in w.columns] - temp_wt = row_w.dot(w) - w = w.apply(lambda x: x/temp_wt - 1, axis=1) - modMat = np.hstack((modMat, w)) - modNames.extend(levels) - weights = np.append(weights, temp_wt) - return modMat[:,1:], modNames[1:], weights[1:] + columnType = np.array([""] * columns) + weights = np.array(np.nan) + for col in range(columns): + id1 = "q" + if matrix.iloc[:, col].dtype == "int": + matrix.iloc[:, col] = matrix.iloc[:, col].apply(float) + elif matrix.iloc[:, col].dtype == "object": + id1 = "f" + columnType[col] = id1 + modMat = np.array([0] * rows).reshape(rows, 1) + modNames = ["0"] + for col in range(columns): + if columnType[col] == "q": + modMat = np.hstack( + (modMat, wt_scale(matrix.iloc[:, col], row_w, bias=1).reshape(rows, 1)) + ) + modNames.append(matrix.columns[col]) + weights = np.append(weights, 1) + elif columnType[col] == "f": + colName = matrix.columns[col] + w = get_dummies(matrix.iloc[:, col]) + levels = [colName + ":" + x for x in w.columns] + temp_wt = row_w.dot(w) + w = w.apply(lambda x: x / temp_wt - 1, axis=1) + modMat = np.hstack((modMat, w)) + modNames.extend(levels) + weights = np.append(weights, temp_wt) + return modMat[:, 1:], modNames[1:], weights[1:] + def ordfunc(mat, wt_col, wt_row, ndim): - X = np.array(mat) - X2 = np.apply_along_axis(lambda x: x*np.sqrt(wt_row), 0, X) - X2 = np.apply_along_axis(lambda x: x*np.sqrt(wt_col), 1, X2) - X2 = X2.T.dot(X2) - evals, evecs = np.linalg.eig(X2) - evals = evals[evals.argsort()[::-1]] - evecs = evecs[:,evals.argsort()[::-1]] - evecs = evecs[:,:ndim] - sds = np.sqrt(evals)[:ndim] - axiswt = 1/np.sqrt(wt_col) - axes = np.apply_along_axis(lambda x: x*axiswt, 0, evecs) - rowcoords = np.apply_along_axis(lambda x: x*wt_col, 1, X) - rowcoords = rowcoords.dot(axes) - colcoords = np.apply_along_axis(lambda x: x*sds, 1, axes) - components = np.apply_along_axis(lambda x: x/sds, 1, rowcoords) - return axes, rowcoords, colcoords, components, evals + X = np.array(mat) + X2 = np.apply_along_axis(lambda x: x * np.sqrt(wt_row), 0, X) + X2 = np.apply_along_axis(lambda x: x * np.sqrt(wt_col), 1, X2) + X2 = X2.T.dot(X2) + evals, evecs = np.linalg.eig(X2) + evals = evals[evals.argsort()[::-1]] + evecs = evecs[:, evals.argsort()[::-1]] + evecs = evecs[:, :ndim] + sds = np.sqrt(evals)[:ndim] + axiswt = 1 / np.sqrt(wt_col) + axes = np.apply_along_axis(lambda x: x * axiswt, 0, evecs) + rowcoords = np.apply_along_axis(lambda x: x * wt_col, 1, X) + rowcoords = rowcoords.dot(axes) + colcoords = np.apply_along_axis(lambda x: x * sds, 1, axes) + components = np.apply_along_axis(lambda x: x / sds, 1, rowcoords) + return axes, rowcoords, colcoords, components, evals + def normalize(X, w): - Z = np.array(X) - norms = np.apply_along_axis(lambda x: np.sqrt(np.sum(x*x*w) / np.sum(w)), 0, Z) - normedMat = np.apply_along_axis(lambda x: x / norms, 1, Z) - normedMat = DataFrame(normedMat, index=X.index) - normedMat.columns = X.columns - return normedMat \ No newline at end of file + Z = np.array(X) + norms = np.apply_along_axis(lambda x: np.sqrt(np.sum(x * x * w) / np.sum(w)), 0, Z) + normedMat = np.apply_along_axis(lambda x: x / norms, 1, Z) + normedMat = DataFrame(normedMat, index=X.index) + normedMat.columns = X.columns + return normedMat diff --git a/ecopy/matrix_comp/simper.py b/ecopy/matrix_comp/simper.py index aab4aee..1200384 100644 --- a/ecopy/matrix_comp/simper.py +++ b/ecopy/matrix_comp/simper.py @@ -1,8 +1,9 @@ import numpy as np from pandas import DataFrame + def simper(data, factor, spNames=None): - ''' + """ Docstring for function ecopy.simper ==================== Conducts a SIMPER (percentage similarity) analysis for a @@ -32,84 +33,80 @@ def simper(data, factor, spNames=None): group1 = data2['Management'] fd = ep.simper(np.array(data1), group1, spNames=data1.columns) print(fd.ix['BF-NM']) - ''' - if not isinstance(data, (DataFrame, np.ndarray)): - msg = 'datamust be either numpy array or dataframe' - raise ValueError(msg) - if isinstance(data, DataFrame): - if (data.dtypes == 'object').any(): - msg = 'DataFrame can only contain numeric values' - raise ValueError(msg) - X = np.array(data).astype('float') - spNames = data.columns - else: - X = data.astype('float') - if np.min(np.sum(X, axis=1))==0: - msg = 'One row is entirely zeros, distance calculations will be meaningless' - raise ValueError(msg) - if (X < 0).any(): - msg = 'Matrix contains negative values' - raise ValueError(msg) - if spNames is None: - spNames = np.arange(X.shape[1]) - s1 = np.array(spNames) - g1 = np.array(factor) - if len(g1) != X.shape[0]: - msg = 'Factor length must equal number of rows in matrix' - raise ValueError(msg) - if len(s1) != X.shape[1]: - msg = 'Species names must equal number of columns in matrix' - raise ValueError(msg) - groupIDs = np.unique(g1) - print('\nComparison indices:') - i = 0 - while i < len(groupIDs)-1: - t1 = X[g1==groupIDs[i],:] - j = i+1 - while j < len(groupIDs): - t2 = X[g1==groupIDs[j],:] - comp = '{0}-{1}'.format(groupIDs[i], groupIDs[j]) - n1 = t1.shape[0] - n2 = t2.shape[0] - deltaI = np.zeros((n1*n2, t1.shape[1])) - k = 0 - while k < n1*n2: - for idx1 in range(n1): - for idx2 in range(n2): - deltaI[k,:] = brayWrap(t1[idx1,:], t2[idx2,:]) - k += 1 - spMeans =np.round(deltaI.mean(axis=0), 2) - spSds = np.round(deltaI.std(axis=0, ddof=1), 2) - spRat = np.round(spMeans/spSds, 2) - spPct = np.round(spMeans/spMeans.sum()*100, 2) - tempDF = DataFrame({'sp_mean': spMeans, - 'sp_sd': spSds, - 'ratio': spRat, - 'sp_pct': spPct}, - index=[[comp]*len(spNames), spNames]) - tempDF.sort_values(by=['sp_pct'], inplace=True, ascending=False) - tempDF['cumulative'] = np.cumsum(tempDF['sp_pct']) - tempDF = tempDF[['sp_mean', 'sp_sd', 'ratio', 'sp_pct', 'cumulative']] - if i==0 and j==i+1: - finalDF = tempDF - else: - finalDF = finalDF.append(tempDF) - print(comp) - j += 1 - i += 1 - return finalDF + """ + if not isinstance(data, (DataFrame, np.ndarray)): + msg = "datamust be either numpy array or dataframe" + raise ValueError(msg) + if isinstance(data, DataFrame): + if (data.dtypes == "object").any(): + msg = "DataFrame can only contain numeric values" + raise ValueError(msg) + X = np.array(data).astype("float") + spNames = data.columns + else: + X = data.astype("float") + if np.min(np.sum(X, axis=1)) == 0: + msg = "One row is entirely zeros, distance calculations will be meaningless" + raise ValueError(msg) + if (X < 0).any(): + msg = "Matrix contains negative values" + raise ValueError(msg) + if spNames is None: + spNames = np.arange(X.shape[1]) + s1 = np.array(spNames) + g1 = np.array(factor) + if len(g1) != X.shape[0]: + msg = "Factor length must equal number of rows in matrix" + raise ValueError(msg) + if len(s1) != X.shape[1]: + msg = "Species names must equal number of columns in matrix" + raise ValueError(msg) + groupIDs = np.unique(g1) + print("\nComparison indices:") + i = 0 + while i < len(groupIDs) - 1: + t1 = X[g1 == groupIDs[i], :] + j = i + 1 + while j < len(groupIDs): + t2 = X[g1 == groupIDs[j], :] + comp = "{0}-{1}".format(groupIDs[i], groupIDs[j]) + n1 = t1.shape[0] + n2 = t2.shape[0] + deltaI = np.zeros((n1 * n2, t1.shape[1])) + k = 0 + while k < n1 * n2: + for idx1 in range(n1): + for idx2 in range(n2): + deltaI[k, :] = brayWrap(t1[idx1, :], t2[idx2, :]) + k += 1 + spMeans = np.round(deltaI.mean(axis=0), 2) + spSds = np.round(deltaI.std(axis=0, ddof=1), 2) + spRat = np.round(spMeans / spSds, 2) + spPct = np.round(spMeans / spMeans.sum() * 100, 2) + tempDF = DataFrame( + {"sp_mean": spMeans, "sp_sd": spSds, "ratio": spRat, "sp_pct": spPct}, + index=[[comp] * len(spNames), spNames], + ) + tempDF.sort_values(by=["sp_pct"], inplace=True, ascending=False) + tempDF["cumulative"] = np.cumsum(tempDF["sp_pct"]) + tempDF = tempDF[["sp_mean", "sp_sd", "ratio", "sp_pct", "cumulative"]] + if i == 0 and j == i + 1: + finalDF = tempDF + else: + finalDF = finalDF.append(tempDF) + print(comp) + j += 1 + i += 1 + return finalDF +def brayWrap(x, y): + temp = np.array([x, y]) + sums = np.sum(temp) + deltas = np.apply_along_axis(brayFunc, 0, temp, sums=sums) + return deltas -def brayWrap(x,y): - temp = np.array([x, y]) - sums = np.sum(temp) - deltas = np.apply_along_axis(brayFunc, 0, temp, sums=sums) - return deltas def brayFunc(m, sums): - delta = np.abs(m[0]-m[1])/sums - return 100*delta - - - + delta = np.abs(m[0] - m[1]) / sums + return 100 * delta diff --git a/ecopy/ordination/correspondance.py b/ecopy/ordination/correspondance.py index c39a6b1..bc4bc9a 100644 --- a/ecopy/ordination/correspondance.py +++ b/ecopy/ordination/correspondance.py @@ -2,8 +2,9 @@ from pandas import DataFrame import matplotlib.pyplot as py + class ca(object): - ''' + """ Docstring for function ecopy.ca ==================== Conducts correspondance analysis (CA). User supplies @@ -63,145 +64,208 @@ class ca(object): bci_ca = ep.ca(BCI) print(bci_ca.summary()) bci_ca.biplot() - ''' - def __init__(self, x, siteNames=None, spNames=None, scaling=1): - # if the data is not a dataframe or array, raise error - if not isinstance(x, (DataFrame, np.ndarray)): - msg = 'Data must either be pandas.DataFrame or nump.ndarray' - raise ValueError(msg) - # if x is a DataFrame - if isinstance(x, DataFrame): - # check NAs - if x.isnull().any().any(): - msg = 'DataFrame contains null values' - raise ValueError(msg) - # check for non-numeric - if (x.dtypes == 'object').any(): - msg = 'DataFrame can only contain numeric values' - raise ValueError(msg) - # convert to a numpy array - y = np.array(x) - # if x is array, simple re-assign - if isinstance(x, np.ndarray): - if np.isnan(x).any(): - msg = 'Array contains null values' - raise ValueError(msg) - y = x - # check for negative values - if y.any() < 0: - msg ='Matrix cannot contain negative values' - raise ValueError(msg) - if scaling not in [1,2]: - msg = 'type parameter must be 1 or 2' - raise ValueError(msg) - if y.shape[0] < y.shape[1]: - y = y.T - self.Trans = True - else: - self.Trans = False - pMat = y.astype('float')/y.sum() - self.w_row = pMat.sum(axis=1) - self.w_col = pMat.sum(axis=0) - w_rowA = self.w_row[:,np.newaxis] - w_colA = self.w_col[np.newaxis,:] - Q = (pMat - w_rowA*w_colA)/np.sqrt(w_rowA*w_colA) - self.evals, self.U = np.linalg.eig(Q.T.dot(Q)) - idx = self.evals.argsort()[::-1] - self.evals = self.evals[idx] - self.U = self.U[:,idx] - self.Uhat = Q.dot(self.U).dot(np.diag(self.evals**-0.5)) - self.evals = self.evals[:-1] - self.U = self.U[:,:-1] - self.Uhat = self.Uhat[:,:-1] - if isinstance(x, DataFrame): - self.siteLabs = x.index - self.spLabs = x.columns - else: - self.siteLabs = ['Site ' + str(x) for x in range(y.shape[0])] - self.spLabs = ['Sp ' + str(x) for x in range(y.shape[1])] - if siteNames is not None: - self.siteLabs = siteNames - if spNames is not None: - self.spLabs = spNames - U2 = self.U.dot(np.diag(self.evals**0.5)) - Uhat2 = self.Uhat.dot(np.diag(self.evals**0.5)) - if self.Trans: - self.cumDesc_Sp = DataFrame(np.apply_along_axis(lambda x: np.cumsum(x**2) / np.sum(x**2), 1, Uhat2)) - self.cumDesc_Site = DataFrame(np.apply_along_axis(lambda x: np.cumsum(x**2) / np.sum(x**2), 1, U2)) - else: - self.cumDesc_Sp = DataFrame(np.apply_along_axis(lambda x: np.cumsum(x**2) / np.sum(x**2), 1, U2)) - self.cumDesc_Site = DataFrame(np.apply_along_axis(lambda x: np.cumsum(x**2) / np.sum(x**2), 1, Uhat2)) - if isinstance(x, DataFrame): - self.cumDesc_Sp.index = x.columns - self.cumDesc_Site.index = x.index - self.cumDesc_Sp.columns = ['CA Axis ' + str(x) for x in range(1, len(self.evals) + 1)] - self.cumDesc_Site.columns = ['CA Axis ' + str(x) for x in range(1, len(self.evals) + 1)] - V = np.diag(self.w_col**-0.5).dot(self.U) - Vhat = np.diag(self.w_row**-0.5).dot(self.Uhat) - F = Vhat.dot(np.diag(self.evals**0.5)) - Fhat = V.dot(np.diag(self.evals**0.5)) - if self.Trans: - siteCent = Fhat - spCent = F - siteOut = V - spOut = Vhat - if scaling==1: - self.siteScores = DataFrame(siteCent, index=self.siteLabs) - self.spScores = DataFrame(spOut, index=self.spLabs) - elif scaling==2: - self.siteScores = DataFrame(siteOut, columns=self.siteLabs) - self.spScores = DataFrame(spCent, columns=self.spLabs) - else: - siteCent = F - spCent = Fhat - siteOut = Vhat - spOut = V - if scaling==1: - self.siteScores = DataFrame(siteCent, index=self.siteLabs) - self.spScores = DataFrame(spOut, index=self.spLabs) - elif scaling==2: - self.siteScores = DataFrame(siteOut, index=self.siteLabs) - self.spScores = DataFrame(spCent, index=self.spLabs) - - - def summary(self): - sds = np.sqrt(self.evals) - props = self.evals / np.sum(self.evals) - cumSums = np.cumsum(self.evals) / np.sum(self.evals) - colNames = ['CA Axis ' + str(x) for x in range(1, len(self.evals)+1)] - sumTable = DataFrame(np.vstack((sds, props, cumSums)), index=['Inertia', 'Prop.', 'Cum. Prop.']) - sumTable.columns = colNames - return sumTable - - def biplot(self, xax=1, yax=2, showSp=True, showSite=True, spCol='r', siteCol='k', spSize=12, siteSize=12, xlim=None, ylim=None): - f, ax = py.subplots() - if showSite: - ax.plot(self.siteScores.iloc[:,xax-1], self.siteScores.iloc[:,yax-1], 'ko', ms=0) - [ax.text(x, y, s, fontsize=siteSize, color=siteCol, ha='center', va='center') for x,y,s in zip(self.siteScores.iloc[:,xax-1], self.siteScores.iloc[:,yax-1], self.siteLabs)] - if showSp: - ax.plot(self.spScores.iloc[:,xax-1], self.spScores.iloc[:,yax-1], 'k^', ms=0) - [ax.text(x,y,s, fontsize=spSize, color=spCol, ha='center', va='center') for x,y,s in zip(self.spScores.iloc[:,xax-1], self.spScores.iloc[:,yax-1], self.spLabs)] - xmax = max(np.amax(self.siteScores.iloc[:,xax-1]), np.amax(self.spScores.iloc[:,xax-1])) - xmin = min(np.amin(self.siteScores.iloc[:,xax-1]), np.amin(self.spScores.iloc[:,xax-1])) - ymax = max(np.amax(self.siteScores.iloc[:,yax-1]), np.amax(self.spScores.iloc[:,yax-1])) - ymin = min(np.min(self.siteScores.iloc[:,yax-1]), np.min(self.spScores.iloc[:,yax-1])) - ax.set_xlim([xmin*1.15, xmax*1.15]) - ax.set_ylim([ymin*1.15, ymax*1.15]) - if xlim is not None: - if not isinstance(xlim, list): - msg = "xlim must be a list" - raise ValueError(msg) - ax.set_xlim(xlim) - if ylim is not None: - if not isinstance(ylim, list): - msg = 'ylim must be a list' - raise ValueError(msg) - ax.set_ylim(ylim) - ax.set_xlabel('CA Axis {!s}'.format(xax)) - ax.set_ylabel('CA Axis {!s}'.format(yax)) - py.show() - + """ + def __init__(self, x, siteNames=None, spNames=None, scaling=1): + # if the data is not a dataframe or array, raise error + if not isinstance(x, (DataFrame, np.ndarray)): + msg = "Data must either be pandas.DataFrame or nump.ndarray" + raise ValueError(msg) + # if x is a DataFrame + if isinstance(x, DataFrame): + # check NAs + if x.isnull().any().any(): + msg = "DataFrame contains null values" + raise ValueError(msg) + # check for non-numeric + if (x.dtypes == "object").any(): + msg = "DataFrame can only contain numeric values" + raise ValueError(msg) + # convert to a numpy array + y = np.array(x) + # if x is array, simple re-assign + if isinstance(x, np.ndarray): + if np.isnan(x).any(): + msg = "Array contains null values" + raise ValueError(msg) + y = x + # check for negative values + if y.any() < 0: + msg = "Matrix cannot contain negative values" + raise ValueError(msg) + if scaling not in [1, 2]: + msg = "type parameter must be 1 or 2" + raise ValueError(msg) + if y.shape[0] < y.shape[1]: + y = y.T + self.Trans = True + else: + self.Trans = False + pMat = y.astype("float") / y.sum() + self.w_row = pMat.sum(axis=1) + self.w_col = pMat.sum(axis=0) + w_rowA = self.w_row[:, np.newaxis] + w_colA = self.w_col[np.newaxis, :] + Q = (pMat - w_rowA * w_colA) / np.sqrt(w_rowA * w_colA) + self.evals, self.U = np.linalg.eig(Q.T.dot(Q)) + idx = self.evals.argsort()[::-1] + self.evals = self.evals[idx] + self.U = self.U[:, idx] + self.Uhat = Q.dot(self.U).dot(np.diag(self.evals ** -0.5)) + self.evals = self.evals[:-1] + self.U = self.U[:, :-1] + self.Uhat = self.Uhat[:, :-1] + if isinstance(x, DataFrame): + self.siteLabs = x.index + self.spLabs = x.columns + else: + self.siteLabs = ["Site " + str(x) for x in range(y.shape[0])] + self.spLabs = ["Sp " + str(x) for x in range(y.shape[1])] + if siteNames is not None: + self.siteLabs = siteNames + if spNames is not None: + self.spLabs = spNames + U2 = self.U.dot(np.diag(self.evals ** 0.5)) + Uhat2 = self.Uhat.dot(np.diag(self.evals ** 0.5)) + if self.Trans: + self.cumDesc_Sp = DataFrame( + np.apply_along_axis( + lambda x: np.cumsum(x ** 2) / np.sum(x ** 2), 1, Uhat2 + ) + ) + self.cumDesc_Site = DataFrame( + np.apply_along_axis(lambda x: np.cumsum(x ** 2) / np.sum(x ** 2), 1, U2) + ) + else: + self.cumDesc_Sp = DataFrame( + np.apply_along_axis(lambda x: np.cumsum(x ** 2) / np.sum(x ** 2), 1, U2) + ) + self.cumDesc_Site = DataFrame( + np.apply_along_axis( + lambda x: np.cumsum(x ** 2) / np.sum(x ** 2), 1, Uhat2 + ) + ) + if isinstance(x, DataFrame): + self.cumDesc_Sp.index = x.columns + self.cumDesc_Site.index = x.index + self.cumDesc_Sp.columns = [ + "CA Axis " + str(x) for x in range(1, len(self.evals) + 1) + ] + self.cumDesc_Site.columns = [ + "CA Axis " + str(x) for x in range(1, len(self.evals) + 1) + ] + V = np.diag(self.w_col ** -0.5).dot(self.U) + Vhat = np.diag(self.w_row ** -0.5).dot(self.Uhat) + F = Vhat.dot(np.diag(self.evals ** 0.5)) + Fhat = V.dot(np.diag(self.evals ** 0.5)) + if self.Trans: + siteCent = Fhat + spCent = F + siteOut = V + spOut = Vhat + if scaling == 1: + self.siteScores = DataFrame(siteCent, index=self.siteLabs) + self.spScores = DataFrame(spOut, index=self.spLabs) + elif scaling == 2: + self.siteScores = DataFrame(siteOut, columns=self.siteLabs) + self.spScores = DataFrame(spCent, columns=self.spLabs) + else: + siteCent = F + spCent = Fhat + siteOut = Vhat + spOut = V + if scaling == 1: + self.siteScores = DataFrame(siteCent, index=self.siteLabs) + self.spScores = DataFrame(spOut, index=self.spLabs) + elif scaling == 2: + self.siteScores = DataFrame(siteOut, index=self.siteLabs) + self.spScores = DataFrame(spCent, index=self.spLabs) + def summary(self): + sds = np.sqrt(self.evals) + props = self.evals / np.sum(self.evals) + cumSums = np.cumsum(self.evals) / np.sum(self.evals) + colNames = ["CA Axis " + str(x) for x in range(1, len(self.evals) + 1)] + sumTable = DataFrame( + np.vstack((sds, props, cumSums)), index=["Inertia", "Prop.", "Cum. Prop."] + ) + sumTable.columns = colNames + return sumTable - \ No newline at end of file + def biplot( + self, + xax=1, + yax=2, + showSp=True, + showSite=True, + spCol="r", + siteCol="k", + spSize=12, + siteSize=12, + xlim=None, + ylim=None, + ): + f, ax = py.subplots() + if showSite: + ax.plot( + self.siteScores.iloc[:, xax - 1], + self.siteScores.iloc[:, yax - 1], + "ko", + ms=0, + ) + [ + ax.text( + x, y, s, fontsize=siteSize, color=siteCol, ha="center", va="center" + ) + for x, y, s in zip( + self.siteScores.iloc[:, xax - 1], + self.siteScores.iloc[:, yax - 1], + self.siteLabs, + ) + ] + if showSp: + ax.plot( + self.spScores.iloc[:, xax - 1], + self.spScores.iloc[:, yax - 1], + "k^", + ms=0, + ) + [ + ax.text(x, y, s, fontsize=spSize, color=spCol, ha="center", va="center") + for x, y, s in zip( + self.spScores.iloc[:, xax - 1], + self.spScores.iloc[:, yax - 1], + self.spLabs, + ) + ] + xmax = max( + np.amax(self.siteScores.iloc[:, xax - 1]), + np.amax(self.spScores.iloc[:, xax - 1]), + ) + xmin = min( + np.amin(self.siteScores.iloc[:, xax - 1]), + np.amin(self.spScores.iloc[:, xax - 1]), + ) + ymax = max( + np.amax(self.siteScores.iloc[:, yax - 1]), + np.amax(self.spScores.iloc[:, yax - 1]), + ) + ymin = min( + np.min(self.siteScores.iloc[:, yax - 1]), + np.min(self.spScores.iloc[:, yax - 1]), + ) + ax.set_xlim([xmin * 1.15, xmax * 1.15]) + ax.set_ylim([ymin * 1.15, ymax * 1.15]) + if xlim is not None: + if not isinstance(xlim, list): + msg = "xlim must be a list" + raise ValueError(msg) + ax.set_xlim(xlim) + if ylim is not None: + if not isinstance(ylim, list): + msg = "ylim must be a list" + raise ValueError(msg) + ax.set_ylim(ylim) + ax.set_xlabel("CA Axis {!s}".format(xax)) + ax.set_ylabel("CA Axis {!s}".format(yax)) + py.show() diff --git a/ecopy/ordination/distance.py b/ecopy/ordination/distance.py index ba04195..d00eb64 100644 --- a/ecopy/ordination/distance.py +++ b/ecopy/ordination/distance.py @@ -1,8 +1,9 @@ import numpy as np -from pandas import DataFrame +from pandas import DataFrame -def distance(x, method='euclidean', transform="1", breakNA=True): - ''' + +def distance(x, method="euclidean", transform="1", breakNA=True): + """ Docstring for function ecopy.distance ======================== Computes a dissimilarity matrix from a given matrix, @@ -95,423 +96,460 @@ def distance(x, method='euclidean', transform="1", breakNA=True): # for binary data varespec[varespec>0] = 1 distance(varespec, method='jaccard) - ''' - listofmethods =['euclidean', 'gow_euclidean', 'simple', 'rogers', 'sokal', 'jaccard', 'sorensen', 'kulczynski', 'bray', 'gower', 'chord', 'manhattan', 'meanChar', 'whittaker', 'canberra', 'hellinger', 'mod_gower', 'ochiai'] - if not isinstance(breakNA, bool): - msg = 'removaNA argument must be boolean' - raise ValueError(msg) - if method not in listofmethods: - msg = 'method argument {0!s} is not an accepted metric'.format(method) - raise ValueError(msg) - if not isinstance(x, (DataFrame, np.ndarray)): - msg = 'x argument must be a numpy array or pandas dataframe' - raise ValueError(msg) - if isinstance(x, DataFrame): - if (x.dtypes == 'object').any(): - msg = 'DataFrame can only contain numeric values' - raise ValueError(msg) - x = np.array(x) - if breakNA: - if np.isnan(np.sum(x)): - msg = 'Matrix contains NA values' - raise ValueError(msg) - if np.min(np.sum(x, axis=1))==0: - msg = 'One row is entirely zeros, distance calculations will be meaningless' - raise ValueError(msg) - if transform not in ['1', 'sqrt']: - msg = 'transform argument must be "1" or "sqrt"' - raise ValueError(msg) - if method in ['simple', 'rogers', 'sokal', 'jaccard', 'sorensen', 'ochiai']: - if np.any((x != 0) & (x != 1)): - msg = 'For method {0}, data must be binary'.format(method) - raise ValueError(msg) - x = x.astype('float') - if method == 'euclidean': - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - distMat[i,j] = eucFunc(x1, x2, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'gow_euclidean': - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,:] - x2 = x[j,:] - delta = ~(np.isnan(x1) + np.isnan(x2)) - delta = delta.astype(int) - x1[np.isnan(x1)] = -999 - x2[np.isnan(x2)] = -999 - distMat[i,j] = eucGow(x1, x2, delta, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'simple': - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - A, B, C, D = matchMat(x1, x2) - distMat[i,j] = simpleSim(A, B, C, D, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'rogers': - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - A, B, C, D = matchMat(x1, x2) - distMat[i,j] = rogerSim(A, B, C, D, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'sokal': - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - A, B, C, D = matchMat(x1, x2) - distMat[i,j] = sokalSim(A, B, C, D, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'jaccard': - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - A, B, C, D = matchMat(x1, x2) - distMat[i,j] = jaccardSim(A, B, C, D, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'sorensen': - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - A, B, C, D = matchMat(x1, x2) - distMat[i,j] = sorenSim(A, B, C, D, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'kulczynski': - if (x<0).any(): - msg = 'Distances are meaningless for negative numbers' - raise ValueError(msg) - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - distMat[i,j] = kulSim(x1, x2, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'bray': - if (x<0).any(): - msg = 'Distances are meaningless for negative numbers' - raise ValueError(msg) - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - distMat[i,j] = braySim(x1, x2, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'gower': - if (x<0).any(): - msg = 'Distances are meaningless for negative numbers' - raise ValueError(msg) - distMat = np.zeros((x.shape[0], x.shape[0])) - R = np.apply_along_axis(lambda z: np.max(z) - np.min(z), 0, x) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - R = R[~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - R = R[~np.isnan(x2)] - distMat[i,j] = gowerSim(x1, x2, R, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'chord': - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - distMat[i,j] = chordDis(x1, x2, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'manhattan': - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - distMat[i,j] = manDist(x1, x2, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'meanChar': - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - distMat[i,j] = charDist(x1, x2, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'whittaker': - if (x<0).any(): - msg = 'Distances are meaningless for negative numbers' - raise ValueError(msg) - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - distMat[i,j] = whitDist(x1, x2, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'canberra': - if (x<0).any(): - msg = 'Distances are meaningless for negative numbers' - raise ValueError(msg) - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - distMat[i,j] = canDist(x1, x2, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'hellinger': - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - distMat[i,j] = chordDis(np.sqrt(x1), np.sqrt(x2), transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'mod_gower': - if (x<0).any(): - msg = 'Distances are meaningless for negative numbers' - raise ValueError(msg) - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - distMat[i,j] = m_gowDist(x1, x2, transform) - distMat[j,i] = distMat[i,j] - return(distMat) - if method == 'ochiai': - distMat = np.zeros((x.shape[0], x.shape[0])) - for i in range(0, distMat.shape[0]): - distMat[i,i] = 0 - for j in range(i+1, distMat.shape[0]): - x1 = x[i,~np.isnan(x[i,:])] - x2 = x[j,~np.isnan(x[i,:])] - x1 = x1[~np.isnan(x2)] - x2 = x2[~np.isnan(x2)] - A, B, C, D = matchMat(x1, x2) - distMat[i,j] = ochiaiSim(A, B, C, D, transform) - distMat[j,i] = distMat[i,j] - return(distMat) + """ + listofmethods = [ + "euclidean", + "gow_euclidean", + "simple", + "rogers", + "sokal", + "jaccard", + "sorensen", + "kulczynski", + "bray", + "gower", + "chord", + "manhattan", + "meanChar", + "whittaker", + "canberra", + "hellinger", + "mod_gower", + "ochiai", + ] + if not isinstance(breakNA, bool): + msg = "removaNA argument must be boolean" + raise ValueError(msg) + if method not in listofmethods: + msg = "method argument {0!s} is not an accepted metric".format(method) + raise ValueError(msg) + if not isinstance(x, (DataFrame, np.ndarray)): + msg = "x argument must be a numpy array or pandas dataframe" + raise ValueError(msg) + if isinstance(x, DataFrame): + if (x.dtypes == "object").any(): + msg = "DataFrame can only contain numeric values" + raise ValueError(msg) + x = np.array(x) + if breakNA: + if np.isnan(np.sum(x)): + msg = "Matrix contains NA values" + raise ValueError(msg) + if np.min(np.sum(x, axis=1)) == 0: + msg = "One row is entirely zeros, distance calculations will be meaningless" + raise ValueError(msg) + if transform not in ["1", "sqrt"]: + msg = 'transform argument must be "1" or "sqrt"' + raise ValueError(msg) + if method in ["simple", "rogers", "sokal", "jaccard", "sorensen", "ochiai"]: + if np.any((x != 0) & (x != 1)): + msg = "For method {0}, data must be binary".format(method) + raise ValueError(msg) + x = x.astype("float") + if method == "euclidean": + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + distMat[i, j] = eucFunc(x1, x2, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "gow_euclidean": + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, :] + x2 = x[j, :] + delta = ~(np.isnan(x1) + np.isnan(x2)) + delta = delta.astype(int) + x1[np.isnan(x1)] = -999 + x2[np.isnan(x2)] = -999 + distMat[i, j] = eucGow(x1, x2, delta, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "simple": + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + A, B, C, D = matchMat(x1, x2) + distMat[i, j] = simpleSim(A, B, C, D, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "rogers": + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + A, B, C, D = matchMat(x1, x2) + distMat[i, j] = rogerSim(A, B, C, D, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "sokal": + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + A, B, C, D = matchMat(x1, x2) + distMat[i, j] = sokalSim(A, B, C, D, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "jaccard": + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + A, B, C, D = matchMat(x1, x2) + distMat[i, j] = jaccardSim(A, B, C, D, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "sorensen": + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + A, B, C, D = matchMat(x1, x2) + distMat[i, j] = sorenSim(A, B, C, D, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "kulczynski": + if (x < 0).any(): + msg = "Distances are meaningless for negative numbers" + raise ValueError(msg) + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + distMat[i, j] = kulSim(x1, x2, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "bray": + if (x < 0).any(): + msg = "Distances are meaningless for negative numbers" + raise ValueError(msg) + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + distMat[i, j] = braySim(x1, x2, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "gower": + if (x < 0).any(): + msg = "Distances are meaningless for negative numbers" + raise ValueError(msg) + distMat = np.zeros((x.shape[0], x.shape[0])) + R = np.apply_along_axis(lambda z: np.max(z) - np.min(z), 0, x) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + R = R[~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + R = R[~np.isnan(x2)] + distMat[i, j] = gowerSim(x1, x2, R, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "chord": + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + distMat[i, j] = chordDis(x1, x2, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "manhattan": + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + distMat[i, j] = manDist(x1, x2, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "meanChar": + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + distMat[i, j] = charDist(x1, x2, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "whittaker": + if (x < 0).any(): + msg = "Distances are meaningless for negative numbers" + raise ValueError(msg) + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + distMat[i, j] = whitDist(x1, x2, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "canberra": + if (x < 0).any(): + msg = "Distances are meaningless for negative numbers" + raise ValueError(msg) + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + distMat[i, j] = canDist(x1, x2, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "hellinger": + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + distMat[i, j] = chordDis(np.sqrt(x1), np.sqrt(x2), transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "mod_gower": + if (x < 0).any(): + msg = "Distances are meaningless for negative numbers" + raise ValueError(msg) + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + distMat[i, j] = m_gowDist(x1, x2, transform) + distMat[j, i] = distMat[i, j] + return distMat + if method == "ochiai": + distMat = np.zeros((x.shape[0], x.shape[0])) + for i in range(0, distMat.shape[0]): + distMat[i, i] = 0 + for j in range(i + 1, distMat.shape[0]): + x1 = x[i, ~np.isnan(x[i, :])] + x2 = x[j, ~np.isnan(x[i, :])] + x1 = x1[~np.isnan(x2)] + x2 = x2[~np.isnan(x2)] + A, B, C, D = matchMat(x1, x2) + distMat[i, j] = ochiaiSim(A, B, C, D, transform) + distMat[j, i] = distMat[i, j] + return distMat + def eucFunc(d1, d2, t): - d = d1-d2 - eucD = np.sqrt(np.sum(np.square(d))) - if t == "1": - return eucD - if t == "sqrt": - return np.sqrt(eucD) + d = d1 - d2 + eucD = np.sqrt(np.sum(np.square(d))) + if t == "1": + return eucD + if t == "sqrt": + return np.sqrt(eucD) + def eucGow(d1, d2, Delta, t): - d = d1-d2 - eucGow = np.sqrt(np.sum(Delta*d**2)/Delta.sum()) - if t == "1": - return eucGow - if t == "sqrt": - return np.sqrt(eucGow) + d = d1 - d2 + eucGow = np.sqrt(np.sum(Delta * d ** 2) / Delta.sum()) + if t == "1": + return eucGow + if t == "sqrt": + return np.sqrt(eucGow) + def matchMat(d1, d2): - A = float(np.sum((d1 == 1) & (d2 == 1))) - B = float(np.sum((d1 == 1) & (d2 == 0))) - C = float(np.sum((d1 == 0) & (d2 == 1))) - D = float(np.sum((d1 == 0) & (d2 == 0))) - return A, B, C, D + A = float(np.sum((d1 == 1) & (d2 == 1))) + B = float(np.sum((d1 == 1) & (d2 == 0))) + C = float(np.sum((d1 == 0) & (d2 == 1))) + D = float(np.sum((d1 == 0) & (d2 == 0))) + return A, B, C, D + def simpleSim(A, B, C, D, t): - S = (A + D) / (A + B + C + D) - if t == "1": - return 1 - S - if t == "sqrt": - return np.sqrt(1 - S) - -def rogerSim(A, B, C ,D, t): - S = (A + D) / (A + 2*B + 2*C + D) - if t == "1": - return 1 - S - if t == "sqrt": - return np.sqrt(1 - S) + S = (A + D) / (A + B + C + D) + if t == "1": + return 1 - S + if t == "sqrt": + return np.sqrt(1 - S) + + +def rogerSim(A, B, C, D, t): + S = (A + D) / (A + 2 * B + 2 * C + D) + if t == "1": + return 1 - S + if t == "sqrt": + return np.sqrt(1 - S) + def sokalSim(A, B, C, D, t): - S = (2*A + 2*D) / (2*A + B + C + 2*D) - if t == "1": - return 1 - S - if t == "sqrt": - return np.sqrt(1 - S) + S = (2 * A + 2 * D) / (2 * A + B + C + 2 * D) + if t == "1": + return 1 - S + if t == "sqrt": + return np.sqrt(1 - S) + def jaccardSim(A, B, C, D, t): - S = A/(A + B + C) - if t == "1": - return 1 - S - if t == "sqrt": - return np.sqrt(1 - S) + S = A / (A + B + C) + if t == "1": + return 1 - S + if t == "sqrt": + return np.sqrt(1 - S) + def sorenSim(A, B, C, D, t): - S = 2*A/(2*A + B + C) - if t == "1": - return 1 - S - if t == "sqrt": - return np.sqrt(1 - S) + S = 2 * A / (2 * A + B + C) + if t == "1": + return 1 - S + if t == "sqrt": + return np.sqrt(1 - S) + def kulSim(d1, d2, t): - A = np.sum(d1) - B = np.sum(d2) - W = np.sum(np.minimum(d1, d2)) - S = 0.5*(W/A + W/B) - if t == "1": - return 1 - S - if t == "sqrt": - return np.sqrt(1 - S) - + A = np.sum(d1) + B = np.sum(d2) + W = np.sum(np.minimum(d1, d2)) + S = 0.5 * (W / A + W / B) + if t == "1": + return 1 - S + if t == "sqrt": + return np.sqrt(1 - S) + + def braySim(d1, d2, t): - A = np.sum(d1) - B = np.sum(d2) - W = np.sum(np.minimum(d1, d2)) - S = (2*W)/(A + B) - if t == "1": - return 1 - S - if t == "sqrt": - return np.sqrt(1 - S) + A = np.sum(d1) + B = np.sum(d2) + W = np.sum(np.minimum(d1, d2)) + S = (2 * W) / (A + B) + if t == "1": + return 1 - S + if t == "sqrt": + return np.sqrt(1 - S) + def gowerSim(d1, d2, R, t): - diffs = 1 - (np.abs(d1-d2)/R) - isabs = d1+d2 ==0 - S = np.sum(~isabs*diffs)/np.sum(~isabs) - if t == "1": - return 1 - S - if t == "sqrt": - return np.sqrt(1 - S) + diffs = 1 - (np.abs(d1 - d2) / R) + isabs = d1 + d2 == 0 + S = np.sum(~isabs * diffs) / np.sum(~isabs) + if t == "1": + return 1 - S + if t == "sqrt": + return np.sqrt(1 - S) + def chordDis(d1, d2, t): - norm1 = 1/(np.sqrt(np.sum(d1**2))) * d1 - norm2 = 1/(np.sqrt(np.sum(d2**2))) * d2 - d = norm1-norm2 - chordD = np.sqrt(np.sum(np.square(d))) - if t == "1": - return chordD - if t == "sqrt": - return np.sqrt(chordD) + norm1 = 1 / (np.sqrt(np.sum(d1 ** 2))) * d1 + norm2 = 1 / (np.sqrt(np.sum(d2 ** 2))) * d2 + d = norm1 - norm2 + chordD = np.sqrt(np.sum(np.square(d))) + if t == "1": + return chordD + if t == "sqrt": + return np.sqrt(chordD) + def manDist(d1, d2, t): - manD = np.sum(np.abs(d1 - d2)) - if t == "1": - return manD - if t == "sqrt": - return np.sqrt(manD) + manD = np.sum(np.abs(d1 - d2)) + if t == "1": + return manD + if t == "sqrt": + return np.sqrt(manD) + def charDist(d1, d2, t): - charD = 1./len(d1) * np.sum(np.abs(d1 - d2)) - if t == "1": - return charD - if t == "sqrt": - return np.sqrt(charD) + charD = 1.0 / len(d1) * np.sum(np.abs(d1 - d2)) + if t == "1": + return charD + if t == "sqrt": + return np.sqrt(charD) + def whitDist(d1, d2, t): - d1 = d1/np.sum(d1) - d2 = d2/np.sum(d2) - whitD = 0.5*sum(np.abs(d1-d2)) - if t == "1": - return whitD - if t == "sqrt": - return np.sqrt(whitD) + d1 = d1 / np.sum(d1) + d2 = d2 / np.sum(d2) + whitD = 0.5 * sum(np.abs(d1 - d2)) + if t == "1": + return whitD + if t == "sqrt": + return np.sqrt(whitD) + def canDist(d1, d2, t): - isabs = d1+d2==0 - d1 = d1[~isabs] - d2 = d2[~isabs] - canD = np.sum(np.abs(d1-d2)/(d1+d2)) * 1./np.sum(~isabs) - if t == "1": - return canD - if t == "sqrt": - return np.sqrt(canD) + isabs = d1 + d2 == 0 + d1 = d1[~isabs] + d2 = d2[~isabs] + canD = np.sum(np.abs(d1 - d2) / (d1 + d2)) * 1.0 / np.sum(~isabs) + if t == "1": + return canD + if t == "sqrt": + return np.sqrt(canD) + def m_gowDist(d1, d2, t): - isabs = d1+d2==0 - charD = 1./np.sum(~isabs) * np.sum(np.abs(d1 - d2)) - if t == "1": - return charD - if t == "sqrt": - return np.sqrt(charD) - + isabs = d1 + d2 == 0 + charD = 1.0 / np.sum(~isabs) * np.sum(np.abs(d1 - d2)) + if t == "1": + return charD + if t == "sqrt": + return np.sqrt(charD) + + def ochiaiSim(A, B, C, D, t): - S = A/np.sqrt((A + B) * (A + C)) + S = A / np.sqrt((A + B) * (A + C)) if t == "1": return 1 - S if t == "sqrt": diff --git a/ecopy/ordination/hillsmith.py b/ecopy/ordination/hillsmith.py index fee759a..1462a4f 100644 --- a/ecopy/ordination/hillsmith.py +++ b/ecopy/ordination/hillsmith.py @@ -3,8 +3,9 @@ import matplotlib.pyplot as plt from ..base_funcs import wt_scale + class hillsmith(object): - ''' + """ Docstring for function ecopy.hillsmith ==================== Conducts ordination on a matrix of mixed data types (both quantitative and qualitative) @@ -55,151 +56,182 @@ class hillsmith(object): dune_env = dune_env[['A1', 'Moisture', 'Manure', 'Use', 'Management']] print(ep.hillsmith(dune_env).summary().iloc[:,:2]) ep.hillsmith(dune_env).biplot(obsNames=False, invert=False) - ''' - def __init__(self, mat, wt_r=None, ndim=2): - - def corfunc(x): - column = x[0] - component = x[1] - if columnType[column]=='q': - w = components[:,component] * wt_r * modMat[:,columnIndex==column].ravel() - return np.sum(w)**2 - else: - x = components[:,component]*wt_r - qual = np.array(df.iloc[:,column]) - groups = np.unique(qual) - denom = [] - num = [] - for g in groups: - denom.append(wt_r[qual==g].sum()) - num.append(x[qual==g].sum()) - denom = np.array(denom) - num = np.array(num) - div = num / denom - return np.sum(denom*div*div) - - def ordfunc(mat, wt_col, wt_row, ndim=2): - X = np.array(mat) - X2 = np.apply_along_axis(lambda x: x*np.sqrt(wt_row), 0, X) - X2 = np.apply_along_axis(lambda x: x*np.sqrt(wt_col), 1, X2) - X2 = X2.T.dot(X2) - evals, evecs = np.linalg.eig(X2) - evals = evals[evals.argsort()[::-1]] - evecs = evecs[:,evals.argsort()[::-1]] - evecs = evecs[:,:ndim] - sds = np.sqrt(evals)[:ndim] - axiswt = 1/np.sqrt(wt_col) - axes = np.apply_along_axis(lambda x: x*axiswt, 0, evecs) - rowcoords = np.apply_along_axis(lambda x: x*wt_col, 1, X) - rowcoords = rowcoords.dot(axes) - colcoords = np.apply_along_axis(lambda x: x*sds, 1, axes) - components = np.apply_along_axis(lambda x: x/sds, 1, rowcoords) - return axes, rowcoords, colcoords, components, evals + """ - if not isinstance(mat, DataFrame): - msg = 'Data must be pandas.DataFrame' - raise ValueError(msg) - if isinstance(mat, DataFrame): - if mat.isnull().any().any(): - msg = 'DataFrame contains null values' - raise ValueError(msg) - df = mat - nr = df.shape[0] - nc = df.shape[1] - if nr < nc: - msg = 'Number of columns cannot exceed number of columns' - raise ValueError(msg) - columnType = np.array([""]*nc) - for column in range(nc): - w1 = 'q' - if df.iloc[:,column].dtype=='int': - df.iloc[:,column] = df.iloc[:,column].apply(float) - elif df.iloc[:,column].dtype=='object': - w1 = 'f' - columnType[column] = w1 - modMat = np.array([0]*nr).reshape(nr, 1) - modNames = ['0'] - if wt_r is None: - wt_r = np.array([1.]*nr) - wt_r = np.array(wt_r) / np.array(wt_r).sum() - wt_c = np.array(np.nan) - columnIndex = np.array(np.nan) - columnID = 0 - for column in range(nc): - if columnType[column]=='q': - modMat = np.hstack((modMat, wt_scale(df.iloc[:,column], wt=wt_r, bias=1).reshape(nr, 1))) - modNames.append(df.columns[column]) - wt_c = np.append(wt_c, 1) - columnIndex = np.append(columnIndex, columnID) - columnID += 1 - elif columnType[column]=='f': - colName = df.columns[column] - w = get_dummies(df.iloc[:,column]) - levels = [colName + ':' + x for x in w.columns] - temp_wt = wt_r.dot(w) - w = w.apply(lambda x: x/temp_wt - 1, axis=1) - wt_c = np.append(wt_c, temp_wt) - modMat = np.hstack((modMat, w)) - modNames.extend(levels) - columnIndex = np.append(columnIndex, np.array([columnID]*len(levels))) - columnID += 1 - modMat = modMat[:,1:] - wt_c = wt_c[1:] - columnIndex = columnIndex[1:] - modNames = modNames[1:] - idMat = np.zeros((nc, ndim, 2), dtype='int') - idMat[:,:,0] = np.repeat(np.arange(nc).reshape(nc, 1), ndim, axis=1) - idMat[:,:,1] = np.repeat(np.arange(ndim).reshape(1, ndim), nc, axis=0) - axes, rowcoords, colcoords, components, evals = ordfunc(modMat, wt_c, wt_r) - corrMat = DataFrame(np.apply_along_axis(corfunc, 2, idMat), index=df.columns) - corrMat.columns = ['Comp1', 'Comp2'] - self.pr_axes = DataFrame(axes, index=modNames) - axis_names = ['Axis {0}'.format(str(i+1)) for i in range(axes.shape[1])] - self.pr_axes.columns = axis_names - self.row_coords = DataFrame(rowcoords, index=df.index) - self.row_coords.columns = axis_names - self.pr_components = DataFrame(components, index=df.index) - component_names = ['Component {0}'.format(str(i+1)) for i in range(components.shape[1])] - self.pr_components.columns = component_names - self.column_coords = DataFrame(colcoords, index=modNames) - self.column_coords.columns = component_names - self.evals = evals + def __init__(self, mat, wt_r=None, ndim=2): + def corfunc(x): + column = x[0] + component = x[1] + if columnType[column] == "q": + w = ( + components[:, component] + * wt_r + * modMat[:, columnIndex == column].ravel() + ) + return np.sum(w) ** 2 + else: + x = components[:, component] * wt_r + qual = np.array(df.iloc[:, column]) + groups = np.unique(qual) + denom = [] + num = [] + for g in groups: + denom.append(wt_r[qual == g].sum()) + num.append(x[qual == g].sum()) + denom = np.array(denom) + num = np.array(num) + div = num / denom + return np.sum(denom * div * div) - def summary(self): - axis_names = ['Axis {0}'.format(str(i+1)) for i in range(len(self.evals))] - sds = np.sqrt(self.evals) - props = self.evals / self.evals.sum() - cumprop = np.cumsum(self.evals) / self.evals.sum() - summDF = DataFrame(np.vstack((sds, props, cumprop)), index=['Std. Dev', 'Prop Var', 'Cum Var']) - summDF.columns = axis_names - return summDF + def ordfunc(mat, wt_col, wt_row, ndim=2): + X = np.array(mat) + X2 = np.apply_along_axis(lambda x: x * np.sqrt(wt_row), 0, X) + X2 = np.apply_along_axis(lambda x: x * np.sqrt(wt_col), 1, X2) + X2 = X2.T.dot(X2) + evals, evecs = np.linalg.eig(X2) + evals = evals[evals.argsort()[::-1]] + evecs = evecs[:, evals.argsort()[::-1]] + evecs = evecs[:, :ndim] + sds = np.sqrt(evals)[:ndim] + axiswt = 1 / np.sqrt(wt_col) + axes = np.apply_along_axis(lambda x: x * axiswt, 0, evecs) + rowcoords = np.apply_along_axis(lambda x: x * wt_col, 1, X) + rowcoords = rowcoords.dot(axes) + colcoords = np.apply_along_axis(lambda x: x * sds, 1, axes) + components = np.apply_along_axis(lambda x: x / sds, 1, rowcoords) + return axes, rowcoords, colcoords, components, evals - def biplot(self, invert=False, xax=1, yax=2, obsNames=True): - points = self.row_coords - arrows = self.pr_axes - if invert: - points = self.column_coords - arrows = self.pr_components - f, ax = plt.subplots() - ax.axvline(0, ls='solid', c='k') - ax.axhline(0, ls='solid', c='k') - if obsNames: - ax.scatter(points.iloc[:,xax-1], points.iloc[:,yax-1], s=0) - for i in range(points.shape[0]): - plt.text(points.iloc[i,xax-1], points.iloc[i,yax-1], points.index.values[i], ha = 'center', va = 'center') - else: - ax.scatter(points.iloc[:,xax-1], points.iloc[:,yax-1]) - for i in range(arrows.shape[0]): - ax.arrow(0, 0, arrows.iloc[i,xax-1], arrows.iloc[i,yax-1], color = 'red', head_width=.05) - ax.text(arrows.iloc[i, xax-1]*1.2, arrows.iloc[i,yax-1]*1.2, arrows.index.values[i], color = 'red', ha = 'center', va = 'center') - xmax = max(np.amax(points.iloc[:,xax-1]), np.amax(arrows.iloc[:,xax-1])) - xmin = min(np.min(points.iloc[:,xax-1]), np.min(arrows.iloc[:,xax-1])) - ymax = max(np.amax(points.iloc[:,yax-1]), np.amax(arrows.iloc[:,yax-1])) - ymin = min(np.amin(points.iloc[:,yax-1]), np.amin(arrows.iloc[:,yax-1])) - ax.set_xlim([xmin + 0.15*xmin, xmax+0.15*xmax]) - ax.set_ylim([ymin + 0.15*ymin, ymax+0.15*ymax]) - ax.set_xlabel('Axis {!s}'.format(xax)) - ax.set_ylabel('Axis {!s}'.format(yax)) - plt.show() + if not isinstance(mat, DataFrame): + msg = "Data must be pandas.DataFrame" + raise ValueError(msg) + if isinstance(mat, DataFrame): + if mat.isnull().any().any(): + msg = "DataFrame contains null values" + raise ValueError(msg) + df = mat + nr = df.shape[0] + nc = df.shape[1] + if nr < nc: + msg = "Number of columns cannot exceed number of columns" + raise ValueError(msg) + columnType = np.array([""] * nc) + for column in range(nc): + w1 = "q" + if df.iloc[:, column].dtype == "int": + df.iloc[:, column] = df.iloc[:, column].apply(float) + elif df.iloc[:, column].dtype == "object": + w1 = "f" + columnType[column] = w1 + modMat = np.array([0] * nr).reshape(nr, 1) + modNames = ["0"] + if wt_r is None: + wt_r = np.array([1.0] * nr) + wt_r = np.array(wt_r) / np.array(wt_r).sum() + wt_c = np.array(np.nan) + columnIndex = np.array(np.nan) + columnID = 0 + for column in range(nc): + if columnType[column] == "q": + modMat = np.hstack( + ( + modMat, + wt_scale(df.iloc[:, column], wt=wt_r, bias=1).reshape(nr, 1), + ) + ) + modNames.append(df.columns[column]) + wt_c = np.append(wt_c, 1) + columnIndex = np.append(columnIndex, columnID) + columnID += 1 + elif columnType[column] == "f": + colName = df.columns[column] + w = get_dummies(df.iloc[:, column]) + levels = [colName + ":" + x for x in w.columns] + temp_wt = wt_r.dot(w) + w = w.apply(lambda x: x / temp_wt - 1, axis=1) + wt_c = np.append(wt_c, temp_wt) + modMat = np.hstack((modMat, w)) + modNames.extend(levels) + columnIndex = np.append(columnIndex, np.array([columnID] * len(levels))) + columnID += 1 + modMat = modMat[:, 1:] + wt_c = wt_c[1:] + columnIndex = columnIndex[1:] + modNames = modNames[1:] + idMat = np.zeros((nc, ndim, 2), dtype="int") + idMat[:, :, 0] = np.repeat(np.arange(nc).reshape(nc, 1), ndim, axis=1) + idMat[:, :, 1] = np.repeat(np.arange(ndim).reshape(1, ndim), nc, axis=0) + axes, rowcoords, colcoords, components, evals = ordfunc(modMat, wt_c, wt_r) + corrMat = DataFrame(np.apply_along_axis(corfunc, 2, idMat), index=df.columns) + corrMat.columns = ["Comp1", "Comp2"] + self.pr_axes = DataFrame(axes, index=modNames) + axis_names = ["Axis {0}".format(str(i + 1)) for i in range(axes.shape[1])] + self.pr_axes.columns = axis_names + self.row_coords = DataFrame(rowcoords, index=df.index) + self.row_coords.columns = axis_names + self.pr_components = DataFrame(components, index=df.index) + component_names = [ + "Component {0}".format(str(i + 1)) for i in range(components.shape[1]) + ] + self.pr_components.columns = component_names + self.column_coords = DataFrame(colcoords, index=modNames) + self.column_coords.columns = component_names + self.evals = evals + def summary(self): + axis_names = ["Axis {0}".format(str(i + 1)) for i in range(len(self.evals))] + sds = np.sqrt(self.evals) + props = self.evals / self.evals.sum() + cumprop = np.cumsum(self.evals) / self.evals.sum() + summDF = DataFrame( + np.vstack((sds, props, cumprop)), index=["Std. Dev", "Prop Var", "Cum Var"] + ) + summDF.columns = axis_names + return summDF + def biplot(self, invert=False, xax=1, yax=2, obsNames=True): + points = self.row_coords + arrows = self.pr_axes + if invert: + points = self.column_coords + arrows = self.pr_components + f, ax = plt.subplots() + ax.axvline(0, ls="solid", c="k") + ax.axhline(0, ls="solid", c="k") + if obsNames: + ax.scatter(points.iloc[:, xax - 1], points.iloc[:, yax - 1], s=0) + for i in range(points.shape[0]): + plt.text( + points.iloc[i, xax - 1], + points.iloc[i, yax - 1], + points.index.values[i], + ha="center", + va="center", + ) + else: + ax.scatter(points.iloc[:, xax - 1], points.iloc[:, yax - 1]) + for i in range(arrows.shape[0]): + ax.arrow( + 0, + 0, + arrows.iloc[i, xax - 1], + arrows.iloc[i, yax - 1], + color="red", + head_width=0.05, + ) + ax.text( + arrows.iloc[i, xax - 1] * 1.2, + arrows.iloc[i, yax - 1] * 1.2, + arrows.index.values[i], + color="red", + ha="center", + va="center", + ) + xmax = max(np.amax(points.iloc[:, xax - 1]), np.amax(arrows.iloc[:, xax - 1])) + xmin = min(np.min(points.iloc[:, xax - 1]), np.min(arrows.iloc[:, xax - 1])) + ymax = max(np.amax(points.iloc[:, yax - 1]), np.amax(arrows.iloc[:, yax - 1])) + ymin = min(np.amin(points.iloc[:, yax - 1]), np.amin(arrows.iloc[:, yax - 1])) + ax.set_xlim([xmin + 0.15 * xmin, xmax + 0.15 * xmax]) + ax.set_ylim([ymin + 0.15 * ymin, ymax + 0.15 * ymax]) + ax.set_xlabel("Axis {!s}".format(xax)) + ax.set_ylabel("Axis {!s}".format(yax)) + plt.show() diff --git a/ecopy/ordination/mds.py b/ecopy/ordination/mds.py index 82a464b..ba664d5 100644 --- a/ecopy/ordination/mds.py +++ b/ecopy/ordination/mds.py @@ -6,8 +6,9 @@ from ..regression import isotonic from ..ordination import pca, pcoa + class MDS(object): - ''' + """ Docstring for function ecopy.pca ==================== Conducts multidimensional scaling (MDS). User @@ -78,295 +79,404 @@ class MDS(object): dunesMDS = ep.MDS(dunes_D, transform='monotone') dunesMDS.biplot() dunesMDS.biplot(descriptors=dunes_T) - ''' - def __init__(self, distmat, siteNames=None, naxes=2, transform='monotone', - ntry=20, tolerance=1E-4, maxiter=3000, init=None): - if transform not in ['monotone', 'absolute', 'linear', 'ratio']: - msg = 'transform must be one of monotone, absolute, linear, ratio' - raise ValueError(msg) - if not isinstance(distmat, (np.ndarray, DataFrame)): - msg = 'distmat must be a square symmetric numpy.ndarray or pandas.DataFrame' - raise ValueError(msg) - if isinstance(distmat, DataFrame): - if (distmat.dtypes == 'object').any(): - msg = 'DataFrame can only contain numeric values' - raise ValueError(msg) - distmat = np.array(distmat) - if distmat.any() < 0: - msg ='Distance matrix cannot contain negative values' - raise ValueError(msg) - if distmat.shape[0] != distmat.shape[1]: - msg = 'distmat must be a square, symmetric distance matrix' - raise ValueError(msg) - if not np.allclose(distmat.T, distmat): - msg ='distmat must be a square, symmetric distance matrix' - raise ValueError(msg) - if siteNames is not None: - self.siteLabs = siteNames - else: - self.siteLabs = ['Site ' + str(x) for x in range(1, distmat.shape[0]+1)] - weights = np.ones(distmat.shape) - weights[np.isnan(distmat)] = 0 - coordStore = None - stressStore = 1 - if init is None: - init2 = pcoa(distmat).U[:,:naxes] - else: - init2 = init - if transform is 'absolute': - Vp = VTrans(weights) - for i in range(ntry): - if i ==0: - Z = init2 - else: - Z = np.random.rand(distmat.shape[0]*naxes).reshape(distmat.shape[0], naxes) - stress1 = 1 - k = 0 - e = 1E4 - while k < maxiter and e > tolerance: - stress2, Xu = absMDS(distmat, Z, weights, Vp) - e = stress1 - stress2 - Z = Xu - stress1 = stress2 - k += 1 - print('Finished at iteration {0}. Stress = {1}'.format(k, stress2)) - if stress2 < stressStore: - stressStore = stress2 - coordStore = Z - self.parameters = None - if transform is 'ratio': - bStore = None - Vp = VTrans(weights) - for i in range(ntry): - if i ==0: - Z = init2 - else: - Z = np.random.rand(distmat.shape[0]*naxes).reshape(distmat.shape[0], naxes) - stress1 = 1 - k = 0 - e = 1E4 - b = np.random.rand(1) - while k < maxiter and e > tolerance: - stress2, Xu, b = ratioMDS(distmat, b, Z, weights, Vp) - e = stress1 - stress2 - Z = Xu - stress1 = stress2 - k += 1 - print('Finished at iteration {0}. Stress = {1}'.format(k, stress2)) - if stress2 < stressStore: - stressStore = stress2 - coordStore = Z - bStore = b - self.parameters = {'b': bStore} - if transform is 'linear': - aStore = None - bStore = None - for i in range(ntry): - if i ==0: - Z = init2 - else: - Z = np.random.rand(distmat.shape[0]*naxes).reshape(distmat.shape[0], naxes) - stress1 = 1 - k = 0 - e = 1E4 - a = np.random.rand(1) - b = np.random.rand(1) - while k < maxiter and e > tolerance: - stress2, Xu, a, b, = linMDS(distmat, a, b, Z) - e = stress1 - stress2 - Z = pca(Xu).scores - stress1 = stress2 - k += 1 - print('Finished at iteration {0}. Stress = {1}'.format(k, stress2)) - if stress2 < stressStore: - stressStore = stress2 - coordStore = Z - aStore = a - bStore = b - self.parameters = {'a': aStore, 'b': bStore} - if transform is 'monotone': - Vp = VTrans(weights) - for i in range(ntry): - if i ==0: - Z = init2 - else: - Z = np.random.rand(distmat.shape[0]*naxes).reshape(distmat.shape[0], naxes) - stress1 = 1 - k = 0 - e = 1E4 - while k < maxiter and e > tolerance: - stress2, Xu = nMDS(distmat, Z, weights, Vp) - e = stress1 - stress2 - Z = pca(Xu).scores - stress1 = stress2 - k += 1 - print('Finished at iteration {0}. Stress = {1}'.format(k, stress2)) - if stress2 < stressStore: - stressStore = stress2 - coordStore = Z - self.scores = np.apply_along_axis(lambda x: x - np.nanmean(x), 0, coordStore) - self.stress = np.min(stressStore) - self.scores = scoreTrans(self.scores, distmat) - self.scores = pca(self.scores).scores - self.obs = distmat - self.transform = transform - print('Final Stress = {0}'.format(self.stress)) + """ + + def __init__( + self, + distmat, + siteNames=None, + naxes=2, + transform="monotone", + ntry=20, + tolerance=1e-4, + maxiter=3000, + init=None, + ): + if transform not in ["monotone", "absolute", "linear", "ratio"]: + msg = "transform must be one of monotone, absolute, linear, ratio" + raise ValueError(msg) + if not isinstance(distmat, (np.ndarray, DataFrame)): + msg = "distmat must be a square symmetric numpy.ndarray or pandas.DataFrame" + raise ValueError(msg) + if isinstance(distmat, DataFrame): + if (distmat.dtypes == "object").any(): + msg = "DataFrame can only contain numeric values" + raise ValueError(msg) + distmat = np.array(distmat) + if distmat.any() < 0: + msg = "Distance matrix cannot contain negative values" + raise ValueError(msg) + if distmat.shape[0] != distmat.shape[1]: + msg = "distmat must be a square, symmetric distance matrix" + raise ValueError(msg) + if not np.allclose(distmat.T, distmat): + msg = "distmat must be a square, symmetric distance matrix" + raise ValueError(msg) + if siteNames is not None: + self.siteLabs = siteNames + else: + self.siteLabs = ["Site " + str(x) for x in range(1, distmat.shape[0] + 1)] + weights = np.ones(distmat.shape) + weights[np.isnan(distmat)] = 0 + coordStore = None + stressStore = 1 + if init is None: + init2 = pcoa(distmat).U[:, :naxes] + else: + init2 = init + if transform is "absolute": + Vp = VTrans(weights) + for i in range(ntry): + if i == 0: + Z = init2 + else: + Z = np.random.rand(distmat.shape[0] * naxes).reshape( + distmat.shape[0], naxes + ) + stress1 = 1 + k = 0 + e = 1e4 + while k < maxiter and e > tolerance: + stress2, Xu = absMDS(distmat, Z, weights, Vp) + e = stress1 - stress2 + Z = Xu + stress1 = stress2 + k += 1 + print("Finished at iteration {0}. Stress = {1}".format(k, stress2)) + if stress2 < stressStore: + stressStore = stress2 + coordStore = Z + self.parameters = None + if transform is "ratio": + bStore = None + Vp = VTrans(weights) + for i in range(ntry): + if i == 0: + Z = init2 + else: + Z = np.random.rand(distmat.shape[0] * naxes).reshape( + distmat.shape[0], naxes + ) + stress1 = 1 + k = 0 + e = 1e4 + b = np.random.rand(1) + while k < maxiter and e > tolerance: + stress2, Xu, b = ratioMDS(distmat, b, Z, weights, Vp) + e = stress1 - stress2 + Z = Xu + stress1 = stress2 + k += 1 + print("Finished at iteration {0}. Stress = {1}".format(k, stress2)) + if stress2 < stressStore: + stressStore = stress2 + coordStore = Z + bStore = b + self.parameters = {"b": bStore} + if transform is "linear": + aStore = None + bStore = None + for i in range(ntry): + if i == 0: + Z = init2 + else: + Z = np.random.rand(distmat.shape[0] * naxes).reshape( + distmat.shape[0], naxes + ) + stress1 = 1 + k = 0 + e = 1e4 + a = np.random.rand(1) + b = np.random.rand(1) + while k < maxiter and e > tolerance: + stress2, Xu, a, b, = linMDS(distmat, a, b, Z) + e = stress1 - stress2 + Z = pca(Xu).scores + stress1 = stress2 + k += 1 + print("Finished at iteration {0}. Stress = {1}".format(k, stress2)) + if stress2 < stressStore: + stressStore = stress2 + coordStore = Z + aStore = a + bStore = b + self.parameters = {"a": aStore, "b": bStore} + if transform is "monotone": + Vp = VTrans(weights) + for i in range(ntry): + if i == 0: + Z = init2 + else: + Z = np.random.rand(distmat.shape[0] * naxes).reshape( + distmat.shape[0], naxes + ) + stress1 = 1 + k = 0 + e = 1e4 + while k < maxiter and e > tolerance: + stress2, Xu = nMDS(distmat, Z, weights, Vp) + e = stress1 - stress2 + Z = pca(Xu).scores + stress1 = stress2 + k += 1 + print("Finished at iteration {0}. Stress = {1}".format(k, stress2)) + if stress2 < stressStore: + stressStore = stress2 + coordStore = Z + self.scores = np.apply_along_axis(lambda x: x - np.nanmean(x), 0, coordStore) + self.stress = np.min(stressStore) + self.scores = scoreTrans(self.scores, distmat) + self.scores = pca(self.scores).scores + self.obs = distmat + self.transform = transform + print("Final Stress = {0}".format(self.stress)) + + def biplot( + self, + xax=1, + yax=2, + siteNames=True, + coords=False, + descriptors=None, + descripNames=None, + spCol="r", + siteCol="k", + spSize=12, + siteSize=12, + ): + if descriptors is not None: + wts = np.diag(descriptors.sum(axis=0) ** -1) + dScores = wts.dot(descriptors.T).dot(self.scores) + if descripNames is None: + if isinstance(descriptors, DataFrame): + descripNames = descriptors.columns + else: + descripNames = [ + "Species {0}".format(i) for i in range(descriptors.shape[1]) + ] + if not coords: + f, ax = plt.subplots() + ax.axvline(0, ls="solid", c="k") + ax.axhline(0, ls="solid", c="k") + if siteNames: + ax.plot(self.scores[:, xax - 1], self.scores[:, yax - 1], "ko", ms=0) + [ + ax.text( + x, + y, + s, + color=siteCol, + fontsize=siteSize, + ha="center", + va="center", + ) + for x, y, s in zip( + self.scores[:, xax - 1], self.scores[:, yax - 1], self.siteLabs + ) + ] + else: + ax.plot(self.scores[:, xax - 1], self.scores[:, yax - 1], "ko", ms=8) + if descriptors is not None: + [ + ax.text( + x, y, s, color=spCol, fontsize=spSize, ha="center", va="center" + ) + for x, y, s in zip(dScores[:, 0], dScores[:, 1], descripNames) + ] + ax.set_xlabel("nMDS Axis {!s}".format(xax)) + ax.set_ylabel("nMDS Axis {!s}".format(yax)) + plt.show() + else: + if descriptors is None: + coordDict = {"Objects": self.scores} + else: + coordDict = {"Objects": self.scores, "Descriptors": dScores} + return coordDict - def biplot(self, xax=1, yax=2, siteNames=True, coords=False, descriptors=None, descripNames=None, spCol='r', siteCol='k', spSize=12, siteSize=12): - if descriptors is not None: - wts = np.diag(descriptors.sum(axis=0)**-1) - dScores = wts.dot(descriptors.T).dot(self.scores) - if descripNames is None: - if isinstance(descriptors, DataFrame): - descripNames = descriptors.columns - else: - descripNames = ['Species {0}'.format(i) for i in range(descriptors.shape[1])] - if not coords: - f, ax = plt.subplots() - ax.axvline(0, ls='solid', c='k') - ax.axhline(0, ls='solid', c='k') - if siteNames: - ax.plot(self.scores[:,xax-1], self.scores[:,yax-1], 'ko', ms=0) - [ax.text(x,y,s, color=siteCol, fontsize=siteSize, ha='center', va='center') for x,y,s in zip(self.scores[:,xax-1], self.scores[:,yax-1], self.siteLabs)] - else: - ax.plot(self.scores[:,xax-1], self.scores[:,yax-1], 'ko', ms=8) - if descriptors is not None: - [ax.text(x,y,s, color=spCol, fontsize=spSize, ha='center', va='center') for x,y,s in zip(dScores[:,0], dScores[:,1], descripNames)] - ax.set_xlabel('nMDS Axis {!s}'.format(xax)) - ax.set_ylabel('nMDS Axis {!s}'.format(yax)) - plt.show() - else: - if descriptors is None: - coordDict = {'Objects': self.scores} - else: - coordDict = {'Objects': self.scores, 'Descriptors': dScores} - return coordDict + def shepard(self): + dHats = np.tril(eucD(self.scores)).ravel() + dObs = np.tril(self.obs).ravel() + f, ax = plt.subplots() + ax.plot(dObs, dHats, "ro") + if self.transform is "linear": + X = np.column_stack((np.ones(len(dObs)), dObs)) + Y = dHats + B = np.linalg.solve(X.T.dot(X), X.T.dot(Y)) + linPreds = B[0] + B[1] * dObs + R2 = 1 - ((dHats - linPreds) ** 2).sum() / ((dHats - dObs) ** 2).sum() + o = dObs.argsort() + ax.plot(dObs[o], linPreds[o], c="b", lw=2) + ax.text( + 0.05, + 0.85, + "R2 = {:.3}".format(R2), + ha="left", + va="center", + transform=ax.transAxes, + ) + if self.transform is "monotone": + monoPreds = isotonic(dHats, dObs) + R2 = ( + 1 + - ((dHats - monoPreds.prediction) ** 2).sum() + / ((dHats - dObs) ** 2).sum() + ) + o = dObs.argsort() + ax.step(dObs[o], monoPreds.prediction[o], c="b", lw=2) + ax.text( + 0.05, + 0.85, + "R2 = {:.3}".format(R2), + ha="left", + va="center", + transform=ax.transAxes, + ) + ax.text( + 0.05, + 0.9, + "Stress = {:.3}".format(self.stress), + ha="left", + va="center", + transform=ax.transAxes, + ) + plt.show() - def shepard(self): - dHats = np.tril(eucD(self.scores)).ravel() - dObs = np.tril(self.obs).ravel() - f, ax = plt.subplots() - ax.plot(dObs, dHats, 'ro') - if self.transform is 'linear': - X = np.column_stack((np.ones(len(dObs)), dObs)) - Y = dHats - B = np.linalg.solve(X.T.dot(X), X.T.dot(Y)) - linPreds = B[0] + B[1]*dObs - R2 = 1 - ((dHats - linPreds)**2).sum() / ((dHats-dObs)**2).sum() - o = dObs.argsort() - ax.plot(dObs[o], linPreds[o], c='b', lw=2) - ax.text(0.05, 0.85, 'R2 = {:.3}'.format(R2), ha='left', va='center', transform=ax.transAxes) - if self.transform is 'monotone': - monoPreds = isotonic(dHats, dObs) - R2 = 1 - ((dHats - monoPreds.prediction)**2).sum() / ((dHats-dObs)**2).sum() - o = dObs.argsort() - ax.step(dObs[o], monoPreds.prediction[o], c='b', lw=2) - ax.text(0.05, 0.85, 'R2 = {:.3}'.format(R2), ha='left', va='center', transform=ax.transAxes) - ax.text(0.05, 0.9, 'Stress = {:.3}'.format(self.stress), ha='left', va='center', transform = ax.transAxes) - plt.show() + def correlations(self): + dFit = eucD(self.scores) + dObs = self.obs + corrs = [] + for i in range(len(dObs)): + corrs.append(pearsonr(dObs[0, :], dFit[0, :])[0]) + corrs = Series(corrs) + return corrs - def correlations(self): - dFit = eucD(self.scores) - dObs = self.obs - corrs = [] - for i in range(len(dObs)): - corrs.append(pearsonr(dObs[0,:], dFit[0,:])[0]) - corrs = Series(corrs) - return corrs + def correlationPlots(self, site=None): + f, ax = plt.subplots() + dFit = eucD(self.scores) + dObs = self.obs + if site is None: + for i in range(len(self.obs)): + ax.plot( + dObs[i, :], + dFit[i, :], + "o", + alpha=0.5, + c=cm.Set1(float(i) / len(dObs), 1), + ) + else: + ax.plot(dObs[site, :], dFit[site, :], "ko", ms=0) + [ + ax.text(x, y, str(s), ha="center", va="center") + for x, y, s in zip( + dObs[site, :], dFit[site, :], range(len(dFit[site, :])) + ) + ] + ax.set_title("Site {0}".format(site)) + ax.set_ylabel("Fitted Distance to Site") + ax.set_xlabel("Observed Distance to Site") + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + ax.yaxis.set_ticks_position("left") + ax.xaxis.set_ticks_position("bottom") + plt.show() - def correlationPlots(self, site=None): - f, ax = plt.subplots() - dFit = eucD(self.scores) - dObs = self.obs - if site is None: - for i in range(len(self.obs)): - ax.plot(dObs[i,:], dFit[i,:], 'o', alpha=0.5, c=cm.Set1(float(i)/len(dObs), 1)) - else: - ax.plot(dObs[site,:], dFit[site,:], 'ko', ms=0) - [ax.text(x,y,str(s), ha='center', va='center') for x,y,s in zip(dObs[site,:], dFit[site,:], range(len(dFit[site,:])))] - ax.set_title('Site {0}'.format(site)) - ax.set_ylabel("Fitted Distance to Site") - ax.set_xlabel("Observed Distance to Site") - ax.spines['top'].set_visible(False) - ax.spines['right'].set_visible(False) - ax.yaxis.set_ticks_position('left') - ax.xaxis.set_ticks_position('bottom') - plt.show() - def VTrans(weights): - V = -weights - np.fill_diagonal(V, 'nan') - np.fill_diagonal(V, np.nansum(-1*V, axis=1)) - return np.linalg.pinv(V) + V = -weights + np.fill_diagonal(V, "nan") + np.fill_diagonal(V, np.nansum(-1 * V, axis=1)) + return np.linalg.pinv(V) + def Bcalc(weights, distmat, dZ): - ratio = weights*distmat/dZ - ratio[dZ==1E-5] = 0 - bZ = -ratio - np.fill_diagonal(bZ, ratio.sum(axis=1)) - return bZ + ratio = weights * distmat / dZ + ratio[dZ == 1e-5] = 0 + bZ = -ratio + np.fill_diagonal(bZ, ratio.sum(axis=1)) + return bZ + def eucD(X): - n = len(X) - one = np.ones((n, 1)) - D = np.sum(X**2, 1).reshape(n, 1).dot(one.T) + one.dot(np.sum(X**2, 1).reshape(1, n)) - 2*X.dot(X.T) - D = np.round(D, 10) - D = np.sqrt(D) - return D + n = len(X) + one = np.ones((n, 1)) + D = ( + np.sum(X ** 2, 1).reshape(n, 1).dot(one.T) + + one.dot(np.sum(X ** 2, 1).reshape(1, n)) + - 2 * X.dot(X.T) + ) + D = np.round(D, 10) + D = np.sqrt(D) + return D + def absMDS(distmat, Z, weights, Vp): - dZ = eucD(Z) - dZ[dZ==0] = 1E-5 - bZ = Bcalc(weights, distmat, dZ) - Xu = Vp.dot(bZ).dot(Z) - dXu = eucD(Xu) - stress = np.sqrt(np.tril(weights*(distmat-dXu)**2).sum() / np.tril(dXu**2).sum()) - return stress, Xu + dZ = eucD(Z) + dZ[dZ == 0] = 1e-5 + bZ = Bcalc(weights, distmat, dZ) + Xu = Vp.dot(bZ).dot(Z) + dXu = eucD(Xu) + stress = np.sqrt( + np.tril(weights * (distmat - dXu) ** 2).sum() / np.tril(dXu ** 2).sum() + ) + return stress, Xu + def ratioMDS(distmat, b, Z, weights, Vp): - dHat = distmat*b - dZ = eucD(Z) - dZ[dZ==0] = 1E-5 - bZ = Bcalc(weights, dHat, dZ) - Xu = Vp.dot(bZ).dot(Z) - dXu = eucD(Xu) - stress = np.sqrt(np.tril(weights*(dHat-dXu)**2).sum() / np.tril(dXu**2).sum()) - b = np.tril(weights*distmat*dXu).sum() / np.tril(weights*distmat**2).sum() - return stress, Xu, b + dHat = distmat * b + dZ = eucD(Z) + dZ[dZ == 0] = 1e-5 + bZ = Bcalc(weights, dHat, dZ) + Xu = Vp.dot(bZ).dot(Z) + dXu = eucD(Xu) + stress = np.sqrt( + np.tril(weights * (dHat - dXu) ** 2).sum() / np.tril(dXu ** 2).sum() + ) + b = np.tril(weights * distmat * dXu).sum() / np.tril(weights * distmat ** 2).sum() + return stress, Xu, b + def linMDS(distmat, a, b, Z): - dHat = a + b*distmat - dZ = eucD(Z) - dZ[dZ==0] = 1E-5 - weights = np.ones(distmat.shape) - weights[np.isnan(distmat)] = 0 - ix = dZ<0 - weights[ix] = weights[ix]*(dZ[ix] + np.abs(dZ[ix]))/dZ[ix] - Vp = VTrans(weights) - bZ = Bcalc(weights, dHat, dZ) - bZ[ix] = 0 - Xu = Vp.dot(bZ).dot(Z) - dXu = eucD(Xu) - stress = np.sqrt(np.tril(weights*(dHat-dXu)**2).sum() / np.tril(dXu**2).sum()) - a = (np.tril(weights*dXu).sum() - b*np.tril(weights*distmat).sum()) / np.tril(weights).sum() - b = (np.tril(weights*distmat*dXu).sum() - a*np.tril(weights*distmat).sum()) / np.tril(weights*distmat**2).sum() - return stress, Xu, a, b + dHat = a + b * distmat + dZ = eucD(Z) + dZ[dZ == 0] = 1e-5 + weights = np.ones(distmat.shape) + weights[np.isnan(distmat)] = 0 + ix = dZ < 0 + weights[ix] = weights[ix] * (dZ[ix] + np.abs(dZ[ix])) / dZ[ix] + Vp = VTrans(weights) + bZ = Bcalc(weights, dHat, dZ) + bZ[ix] = 0 + Xu = Vp.dot(bZ).dot(Z) + dXu = eucD(Xu) + stress = np.sqrt( + np.tril(weights * (dHat - dXu) ** 2).sum() / np.tril(dXu ** 2).sum() + ) + a = (np.tril(weights * dXu).sum() - b * np.tril(weights * distmat).sum()) / np.tril( + weights + ).sum() + b = ( + np.tril(weights * distmat * dXu).sum() - a * np.tril(weights * distmat).sum() + ) / np.tril(weights * distmat ** 2).sum() + return stress, Xu, a, b + def nMDS(distmat, Z, weights, Vp): - dZ = eucD(Z) - dZ[dZ==0] = 1E-5 - diss_f = distmat.ravel() - dhat_f = dZ.ravel() - dhat = isotonic(dhat_f, diss_f) - dhat = dhat.prediction.reshape(distmat.shape) - stress = np.sqrt(np.tril((weights*(dZ - dhat)**2)).sum() / np.tril(weights*dZ**2).sum()) - bZ = Bcalc(weights, dhat, dZ) - Xu = Vp.dot(bZ).dot(Z) - return stress, Xu + dZ = eucD(Z) + dZ[dZ == 0] = 1e-5 + diss_f = distmat.ravel() + dhat_f = dZ.ravel() + dhat = isotonic(dhat_f, diss_f) + dhat = dhat.prediction.reshape(distmat.shape) + stress = np.sqrt( + np.tril((weights * (dZ - dhat) ** 2)).sum() / np.tril(weights * dZ ** 2).sum() + ) + bZ = Bcalc(weights, dhat, dZ) + Xu = Vp.dot(bZ).dot(Z) + return stress, Xu + def scoreTrans(x, distmat): - distMax = eucD(x).max() - obsMax = x.max() - scale = obsMax / distMax - return x*scale + distMax = eucD(x).max() + obsMax = x.max() + scale = obsMax / distMax + return x * scale diff --git a/ecopy/ordination/ord_plot.py b/ecopy/ordination/ord_plot.py index 1fdb46d..fa0b03b 100644 --- a/ecopy/ordination/ord_plot.py +++ b/ecopy/ordination/ord_plot.py @@ -3,8 +3,19 @@ from pandas import DataFrame, Series import matplotlib.pyplot as plt -def ord_plot(x, groups, y=None, colors=None, type='Hull', label=True, showPoints=True, xlab='Axis 1', ylab='Axis 2'): - ''' + +def ord_plot( + x, + groups, + y=None, + colors=None, + type="Hull", + label=True, + showPoints=True, + xlab="Axis 1", + ylab="Axis 2", +): + """ Docstring for function ecopy.ord_plot ==================== Delineates different groups in ordination (or regular) space. @@ -47,75 +58,105 @@ def ord_plot(x, groups, y=None, colors=None, type='Hull', label=True, showPoints # x and y coordinates in different matrices. line plot. ep.ord_plot(x=X, y=Y, groups=GroupID, type='Line', xlab='PC1', ylab='PC2', showPoints=False, label=False) - ''' - if not isinstance(x, (np.ndarray, DataFrame, Series)): - msg = 'x must be a numpy.ndarray, pandas.DataFrame, or pandas.Series' - raise ValueError (msg) - x = np.array(x) - if y is not None: - if not isinstance(y, (np.ndarray, DataFrame, Series)): - msg = 'y must be a numpy.ndarray, pandas.DataFrame, or pandas.Series' - z = np.vstack((x, y)).T - if z.shape[1]!=2: - msg = 'x and y can only have one column if y is provided' - raise ValueError(msg) - else: - if x.shape[1] != 2: - msg = 'x must be a n x 2 matrix' - raise ValueError(msg) - z = x - if type not in ['Hull', 'Line']: - msg = 'type must either be Hull or Line' - raise ValueError(msg) - if not isinstance(groups, (list, DataFrame, Series)): - msg = 'groups must be either a list, pandas.DataFrame, or pandas.Series' - raise ValueError(msg) - g = list(groups) - unique_g = list(set(g)) - ng = len(unique_g) - if colors is None: - cmap = plt.get_cmap('Paired') - cID = [cmap(i) for i in np.linspace(0, 1, ng)] - else: - if not isinstance(colors, (list, Series, DataFrame)): - msg = 'colors must be a string, list, pandas.DataFrame, or pandas.Series' - if isinstance(colors, (list, Series, DataFrame)): - cID = list(colors) - if isinstance(colors, str): - cID = [colors]*ng - if len(cID) != len(unique_g): - msg = 'list of colors must equal number of groups' - raise ValueError(msg) - if type=='Hull': - f, ax = plt.subplots() - for j in range(len(unique_g)): - tempBOOL = np.array(groups)==unique_g[j] - tempX = z[tempBOOL,:] - tempHull = ConvexHull(tempX) - for simplex in tempHull.simplices: - ax.plot(tempX[simplex, 0], tempX[simplex, 1], '-', c=cID[j], lw=1) - if showPoints: - ax.plot(tempX[:,0], tempX[:,1], 'o', mfc=cID[j], mew=1, label=unique_g[j]) - if label: - Xcent = tempX[:,0].mean() - Ycent = tempX[:,1].mean() - ax.text(Xcent, Ycent, unique_g[j], color=cID[j], ha='center', va='center', bbox={'facecolor': 'white', 'pad': 10, 'edgecolor': cID[j], 'lw': 1}) - ax.set_xlabel(xlab) - ax.set_ylabel(ylab) - plt.show() - if type=='Line': - f, ax = plt.subplots() - for j in range(len(unique_g)): - tempBOOL = np.array(groups)==unique_g[j] - tempX = z[tempBOOL,:] - Xcent = tempX[:,0].mean() - Ycent = tempX[:,1].mean() - for l in range(tempX.shape[0]): - ax.plot([Xcent, tempX[l,0]], [Ycent, tempX[l,1]], c=cID[j], lw=1) - if showPoints: - ax.plot(tempX[:,0], tempX[:,1], 'o', mfc=cID[j], mew=1, label=unique_g[j]) - if label: - ax.text(Xcent, Ycent, unique_g[j], color=cID[j], ha='center', va='center', bbox={'facecolor': 'white', 'pad': 10, 'edgecolor': cID[j], 'lw': 1}) - ax.set_xlabel(xlab) - ax.set_ylabel(ylab) - plt.show() + """ + if not isinstance(x, (np.ndarray, DataFrame, Series)): + msg = "x must be a numpy.ndarray, pandas.DataFrame, or pandas.Series" + raise ValueError(msg) + x = np.array(x) + if y is not None: + if not isinstance(y, (np.ndarray, DataFrame, Series)): + msg = "y must be a numpy.ndarray, pandas.DataFrame, or pandas.Series" + z = np.vstack((x, y)).T + if z.shape[1] != 2: + msg = "x and y can only have one column if y is provided" + raise ValueError(msg) + else: + if x.shape[1] != 2: + msg = "x must be a n x 2 matrix" + raise ValueError(msg) + z = x + if type not in ["Hull", "Line"]: + msg = "type must either be Hull or Line" + raise ValueError(msg) + if not isinstance(groups, (list, DataFrame, Series)): + msg = "groups must be either a list, pandas.DataFrame, or pandas.Series" + raise ValueError(msg) + g = list(groups) + unique_g = list(set(g)) + ng = len(unique_g) + if colors is None: + cmap = plt.get_cmap("Paired") + cID = [cmap(i) for i in np.linspace(0, 1, ng)] + else: + if not isinstance(colors, (list, Series, DataFrame)): + msg = "colors must be a string, list, pandas.DataFrame, or pandas.Series" + if isinstance(colors, (list, Series, DataFrame)): + cID = list(colors) + if isinstance(colors, str): + cID = [colors] * ng + if len(cID) != len(unique_g): + msg = "list of colors must equal number of groups" + raise ValueError(msg) + if type == "Hull": + f, ax = plt.subplots() + for j in range(len(unique_g)): + tempBOOL = np.array(groups) == unique_g[j] + tempX = z[tempBOOL, :] + tempHull = ConvexHull(tempX) + for simplex in tempHull.simplices: + ax.plot(tempX[simplex, 0], tempX[simplex, 1], "-", c=cID[j], lw=1) + if showPoints: + ax.plot( + tempX[:, 0], tempX[:, 1], "o", mfc=cID[j], mew=1, label=unique_g[j] + ) + if label: + Xcent = tempX[:, 0].mean() + Ycent = tempX[:, 1].mean() + ax.text( + Xcent, + Ycent, + unique_g[j], + color=cID[j], + ha="center", + va="center", + bbox={ + "facecolor": "white", + "pad": 10, + "edgecolor": cID[j], + "lw": 1, + }, + ) + ax.set_xlabel(xlab) + ax.set_ylabel(ylab) + plt.show() + if type == "Line": + f, ax = plt.subplots() + for j in range(len(unique_g)): + tempBOOL = np.array(groups) == unique_g[j] + tempX = z[tempBOOL, :] + Xcent = tempX[:, 0].mean() + Ycent = tempX[:, 1].mean() + for l in range(tempX.shape[0]): + ax.plot([Xcent, tempX[l, 0]], [Ycent, tempX[l, 1]], c=cID[j], lw=1) + if showPoints: + ax.plot( + tempX[:, 0], tempX[:, 1], "o", mfc=cID[j], mew=1, label=unique_g[j] + ) + if label: + ax.text( + Xcent, + Ycent, + unique_g[j], + color=cID[j], + ha="center", + va="center", + bbox={ + "facecolor": "white", + "pad": 10, + "edgecolor": cID[j], + "lw": 1, + }, + ) + ax.set_xlabel(xlab) + ax.set_ylabel(ylab) + plt.show() diff --git a/ecopy/ordination/pca.py b/ecopy/ordination/pca.py index ad84588..3f549bf 100644 --- a/ecopy/ordination/pca.py +++ b/ecopy/ordination/pca.py @@ -2,8 +2,9 @@ from pandas import DataFrame import matplotlib.pyplot as py + class pca(object): - ''' + """ Docstring for function ecopy.pca ==================== Conducts principle components analysis (PCA). User @@ -52,109 +53,132 @@ class pca(object): prcomp.summary_desc() prcomp.biplot(type = 'distance') prcomp.biplot(type = 'correlation', obsNames = True) - ''' - def __init__(self, x, scale = True, varNames = None): - if not isinstance(x, (DataFrame, np.ndarray)): - msg = 'Data must either be pandas.DataFrame or numpy.ndarray' - raise ValueError(msg) - if isinstance(x, DataFrame): - if x.isnull().any().any(): - msg = 'DataFrame contains null values' - raise ValueError(msg) - if (x.dtypes == 'object').any(): - msg = 'DataFrame can only contain numeric values' - raise ValueError(msg) - y = np.array(x) - if isinstance(x, np.ndarray): - if np.isnan(x).any(): - msg = 'Array contains null values' - raise ValueError(msg) - y = x - if not isinstance(scale, bool): - msg = "scale argument must be boolean" - raise ValueError(msg) - if scale: - y = np.apply_along_axis(standardize, 0, y) - else: - y = np.apply_along_axis(lambda z: z - np.mean(z), 0, y) - self.evals, self.evecs, self.scores = eig_decomp(y) - if isinstance(x, DataFrame): - self.varNames = x.columns - else: - self.varNames = varNames - if isinstance(x, DataFrame): - self.labs = np.array(x.index) - else: - self.labs = range(x.shape[0]) - - - - def summary_imp(self): - sd = np.sqrt(self.evals) - props = self.evals/np.sum(self.evals) - cums = np.cumsum(self.evals)/np.sum(self.evals) - names = ['PC' + str(i) for i in range(1, self.evecs.shape[1]+1)] - imp = DataFrame(np.vstack((sd, props, cums)), index = ['Std Dev', 'Prop Var', 'Cum Var']) - imp.columns = names - return imp - - def summary_rot(self): - names = ['PC' + str(i) for i in range(1, self.evecs.shape[1]+1)] - rot = DataFrame(self.evecs, index=self.varNames) - rot.columns = names - return rot - - def summary_desc(self): - names = ['PC' + str(i) for i in range(1, self.evecs.shape[1]+1)] - U2 = self.evecs.dot(np.diag(self.evals**0.5)) - desCums = np.apply_along_axis(lambda x: np.cumsum(x**2) / np.sum(x**2), 1, U2) - desc = DataFrame(desCums, index=self.varNames) - desc.columns = names - return desc - - def biplot(self, xax=1, yax=2, type='distance', obsNames = False): - if type not in ['distance', 'correlation']: - msg = 'type argument must be either distance or correlation' - raise ValueError(msg) - if type=='distance': - ScorePlot = self.scores - VecPlot = self.evecs - if type=='correlation': - VecPlot = self.evecs.dot(np.diag(self.evals**0.5)) - ScorePlot = self.scores.dot(np.diag(self.evals**-0.5)) - if self.varNames is None: - self.varNames = range(1, self.evecs.shape[0]+1) - f, ax = py.subplots() - ax.axvline(0, ls='solid', c='k') - ax.axhline(0, ls='solid', c='k') - if obsNames: - ax.scatter(ScorePlot[:,xax-1], ScorePlot[:,yax-1], s=0) - for i in range(ScorePlot.shape[0]): - py.text(ScorePlot[i,xax-1], ScorePlot[i,yax-1], self.labs[i], ha = 'center', va = 'center') - else: - ax.scatter(ScorePlot[:,xax-1], ScorePlot[:,yax-1]) - for i in range(VecPlot.shape[0]): - ax.arrow(0, 0, VecPlot[i,xax-1], VecPlot[i,yax-1], color = 'red', head_width=.05) - ax.text(VecPlot[i, xax-1]*1.2, VecPlot[i,yax-1]*1.2, self.varNames[i], color = 'red', ha = 'center', va = 'center') - xmax = max(np.amax(ScorePlot[:,xax-1]), np.amax(VecPlot[:,xax-1])) - xmin = min(np.min(ScorePlot[:,xax-1]), np.min(VecPlot[:,xax-1])) - ymax = max(np.amax(ScorePlot[:,yax-1]), np.amax(VecPlot[:,yax-1])) - ymin = min(np.amin(ScorePlot[:,yax-1]), np.amin(VecPlot[:,yax-1])) - ax.set_xlim([xmin + 0.15*xmin, xmax+0.15*xmax]) - ax.set_ylim([ymin + 0.15*ymin, ymax+0.15*ymax]) - ax.set_xlabel('PC {!s}'.format(xax)) - ax.set_ylabel('PC {!s}'.format(yax)) - py.show() - + """ + + def __init__(self, x, scale=True, varNames=None): + if not isinstance(x, (DataFrame, np.ndarray)): + msg = "Data must either be pandas.DataFrame or numpy.ndarray" + raise ValueError(msg) + if isinstance(x, DataFrame): + if x.isnull().any().any(): + msg = "DataFrame contains null values" + raise ValueError(msg) + if (x.dtypes == "object").any(): + msg = "DataFrame can only contain numeric values" + raise ValueError(msg) + y = np.array(x) + if isinstance(x, np.ndarray): + if np.isnan(x).any(): + msg = "Array contains null values" + raise ValueError(msg) + y = x + if not isinstance(scale, bool): + msg = "scale argument must be boolean" + raise ValueError(msg) + if scale: + y = np.apply_along_axis(standardize, 0, y) + else: + y = np.apply_along_axis(lambda z: z - np.mean(z), 0, y) + self.evals, self.evecs, self.scores = eig_decomp(y) + if isinstance(x, DataFrame): + self.varNames = x.columns + else: + self.varNames = varNames + if isinstance(x, DataFrame): + self.labs = np.array(x.index) + else: + self.labs = range(x.shape[0]) + + def summary_imp(self): + sd = np.sqrt(self.evals) + props = self.evals / np.sum(self.evals) + cums = np.cumsum(self.evals) / np.sum(self.evals) + names = ["PC" + str(i) for i in range(1, self.evecs.shape[1] + 1)] + imp = DataFrame( + np.vstack((sd, props, cums)), index=["Std Dev", "Prop Var", "Cum Var"] + ) + imp.columns = names + return imp + + def summary_rot(self): + names = ["PC" + str(i) for i in range(1, self.evecs.shape[1] + 1)] + rot = DataFrame(self.evecs, index=self.varNames) + rot.columns = names + return rot + + def summary_desc(self): + names = ["PC" + str(i) for i in range(1, self.evecs.shape[1] + 1)] + U2 = self.evecs.dot(np.diag(self.evals ** 0.5)) + desCums = np.apply_along_axis( + lambda x: np.cumsum(x ** 2) / np.sum(x ** 2), 1, U2 + ) + desc = DataFrame(desCums, index=self.varNames) + desc.columns = names + return desc + + def biplot(self, xax=1, yax=2, type="distance", obsNames=False): + if type not in ["distance", "correlation"]: + msg = "type argument must be either distance or correlation" + raise ValueError(msg) + if type == "distance": + ScorePlot = self.scores + VecPlot = self.evecs + if type == "correlation": + VecPlot = self.evecs.dot(np.diag(self.evals ** 0.5)) + ScorePlot = self.scores.dot(np.diag(self.evals ** -0.5)) + if self.varNames is None: + self.varNames = range(1, self.evecs.shape[0] + 1) + f, ax = py.subplots() + ax.axvline(0, ls="solid", c="k") + ax.axhline(0, ls="solid", c="k") + if obsNames: + ax.scatter(ScorePlot[:, xax - 1], ScorePlot[:, yax - 1], s=0) + for i in range(ScorePlot.shape[0]): + py.text( + ScorePlot[i, xax - 1], + ScorePlot[i, yax - 1], + self.labs[i], + ha="center", + va="center", + ) + else: + ax.scatter(ScorePlot[:, xax - 1], ScorePlot[:, yax - 1]) + for i in range(VecPlot.shape[0]): + ax.arrow( + 0, + 0, + VecPlot[i, xax - 1], + VecPlot[i, yax - 1], + color="red", + head_width=0.05, + ) + ax.text( + VecPlot[i, xax - 1] * 1.2, + VecPlot[i, yax - 1] * 1.2, + self.varNames[i], + color="red", + ha="center", + va="center", + ) + xmax = max(np.amax(ScorePlot[:, xax - 1]), np.amax(VecPlot[:, xax - 1])) + xmin = min(np.min(ScorePlot[:, xax - 1]), np.min(VecPlot[:, xax - 1])) + ymax = max(np.amax(ScorePlot[:, yax - 1]), np.amax(VecPlot[:, yax - 1])) + ymin = min(np.amin(ScorePlot[:, yax - 1]), np.amin(VecPlot[:, yax - 1])) + ax.set_xlim([xmin + 0.15 * xmin, xmax + 0.15 * xmax]) + ax.set_ylim([ymin + 0.15 * ymin, ymax + 0.15 * ymax]) + ax.set_xlabel("PC {!s}".format(xax)) + ax.set_ylabel("PC {!s}".format(yax)) + py.show() def standardize(a): - return (a - np.mean(a))/np.std(a, ddof = 1) + return (a - np.mean(a)) / np.std(a, ddof=1) + def eig_decomp(y): - V, W, U = np.linalg.svd(y) - N = y.shape[0] - evecs = U.T - evals = W**2/(N-1) - scores = y.dot(evecs) - return np.real(evals), np.real(evecs), scores \ No newline at end of file + V, W, U = np.linalg.svd(y) + N = y.shape[0] + evecs = U.T + evals = W ** 2 / (N - 1) + scores = y.dot(evecs) + return np.real(evals), np.real(evecs), scores diff --git a/ecopy/ordination/pcoa.py b/ecopy/ordination/pcoa.py index fa0dddc..9777510 100644 --- a/ecopy/ordination/pcoa.py +++ b/ecopy/ordination/pcoa.py @@ -3,8 +3,9 @@ import matplotlib.pyplot as py from warnings import warn + class pcoa(object): - ''' + """ Docstring for function ecopy.pcoa ==================== Conducts principle coordinate analysis of a user-supplied @@ -69,151 +70,180 @@ class pcoa(object): print(pc1.summary()) pc1.biplot() pc1.shepard() - ''' - def __init__(self, x, correction=None, siteNames=None): - if not isinstance(x, (DataFrame, np.ndarray)): - msg = 'Data must either be pandas.DataFrame or nump.ndarray' - raise ValueError(msg) - if isinstance(x, DataFrame): - if (x.dtypes == 'object').any(): - msg = 'DataFrame can only contain numeric values' - raise ValueError(msg) - y = np.array(x) - if isinstance(x, np.ndarray): - y = x - if np.isnan(y).any(): - msg = 'Distance matrix contains null values' - raise ValueError(msg) - if y.any() < 0: - msg ='Distance matrix cannot contain negative values' - raise ValueError(msg) - if y.shape[0] != y.shape[1]: - msg = 'Distance matrix must be square' - raise ValueError(msg) - if not np.allclose(y.T, y): - msg ='Distance matrix must be symmetric' - raise ValueError(msg) - A = -0.5*np.square(y.astype('float')) - n = y.shape[0] - ones = np.ones(n)[np.newaxis].T - I = np.eye(n) - D = (I - ones.dot(ones.T)/n).dot(A).dot(I - ones.dot(ones.T)/n) - self.evals, self.U = np.linalg.eig(D) - self.evals = self.evals.real - self.U = self.U.real - idx = self.evals.argsort()[::-1] - self.U = self.U[:,idx] - if correction is not None: - if correction not in ['1', '2']: - msg = "correction must be either '1' or '2'" - raise ValueError(msg) - if correction is '1': - negEvl = np.abs(np.min(self.evals[self.evals < 0])) - A = -0.5*np.square(y) - negEvl - np.fill_diagonal(A, 0) - D = (I - ones.dot(ones.T)/n).dot(A).dot(I - ones.dot(ones.T)/n) - self.evals, self.U = np.linalg.eig(D) - idx = self.evals.argsort()[::-1] - self.U = self.U[:,idx] - self.correction = negEvl - if correction is '2': - mat0 = np.zeros((n,n)) - matI = -1.*np.eye(n) - d1 = 2.*D - d2 = -0.5*y - d2 = -4.*(I - ones.dot(ones.T)/n).dot(d2).dot(I - ones.dot(ones.T)/n) - t1 = np.concatenate((mat0, matI), axis=0) - t2 = np.concatenate((d1, d2), axis=0) - specMat = np.concatenate((t1, t2), axis=1) - t_evals, t_evecs =np.linalg.eig(specMat) - posEvl = np.max(np.real(t_evals)) - A = -0.5*(np.square(y.astype('float') + posEvl)) - np.fill_diagonal(A, 0) - D = (I - ones.dot(ones.T)/n).dot(A).dot(I - ones.dot(ones.T)/n) - self.evals, self.U = np.linalg.eig(D) - idx = self.evals.argsort()[::-1] - self.U = self.U[:,idx] - self.correction = posEvl - self.evals = np.round(self.evals[idx], 4) - self.U = np.round(self.U.dot(np.diag(np.sqrt(self.evals))), 4) - self.siteLabs = ['Site ' + str(x) for x in range(1, y.shape[0]+1)] - if isinstance(x, DataFrame): - self.siteLabs = x.index - if siteNames is not None: - self.siteLabs = siteNames - self.y2 = y - - def summary(self): - sds = np.sqrt(self.evals) - props = self.evals / np.sum(self.evals) - cumSums = np.cumsum(self.evals) / np.sum(self.evals) - colNames = ['PCoA Axis ' + str(x) for x in range(1, len(self.evals)+1)] - sumTable = DataFrame(np.vstack((sds, props, cumSums)), index=['Std. Dev', 'Prop.', 'Cum. Prop.']) - sumTable.columns = colNames - return sumTable - - def biplot(self, coords=False, xax=1, yax=2, descriptors=None, descripNames=None, spCol='r', siteCol='k', spSize=12, siteSize=12): - if descriptors is not None: - warn('\nWarning: Descriptors must not be binary.\nIgnore this message if all descriptors are quantitative\n') - if not isinstance(descriptors, (DataFrame, np.ndarray)): - msg = 'descriptors must be a pandas.DataFrame or numpy.ndarray' - raise ValueError(msg) - if isinstance(descriptors, DataFrame): - if (descriptors.dtypes == 'object').any(): - msg = 'DataFrame can only contain numeric values' - raise ValueError(msg) - d2 = np.array(descriptors) - d2 = np.apply_along_axis(lambda x: (x - np.mean(x))/np.std(x, ddof=1), 0, d2) - if isinstance(descriptors, DataFrame): - dLabs = descriptors.columns.values - elif descripNames is not None: - if len(descripNames) != d2.shape[1]: - msg = 'descripNames must be equal to the number of columns in descriptors' - raise ValueError(msg) - dLabs = descripNames - else: - dLabs = ['D'+str(x) for x in range(1, d2.shape[1] + 1)] - U2 = self.U[:,[xax-1, yax-1]].astype('float') - U2 = np.apply_along_axis(lambda x: (x - np.mean(x))/np.std(x, ddof=1), 0, U2) - S = (1./(d2.shape[0]-1))*d2.T.dot(U2) - dProj = np.sqrt(d2.shape[0]-1)*S.dot(np.diag(self.evals[[xax-1, yax-2]]**-0.5)) - if not coords: - f, ax = py.subplots() - ax.axvline(0, ls='solid', c='k') - ax.axhline(0, ls='solid', c='k') - ax.plot(self.U[:,xax-1], self.U[:,yax-1], 'ko', ms=0) - [ax.text(x,y,s, color=siteCol, fontsize=siteSize, ha='center', va='center') for x,y,s in zip(self.U[:,xax-1], self.U[:,yax-1], self.siteLabs)] - if descriptors is not None: - ax.plot(dProj[:,0], dProj[:,1], 'ko', ms=0) - [ax.text(x,y,s, color=spCol, fontsize=spSize, ha='center', va='center') for x,y,s in zip(dProj[:,0], dProj[:,1], dLabs)] - ax.set_xlabel('PCoA Axis {!s}'.format(xax)) - ax.set_ylabel('PCoA Axis {!s}'.format(yax)) - py.show() - else: - if descriptors is not None: - return {'Objects': self.U[:,[xax-1, yax-1]], 'Descriptors': dProj} - else: - return {'Objects': self.U[:,[xax-1, yax-1]]} - - def shepard(self, xax=1, yax=2): - coords = self.U[:,[xax-1, yax-1]] - reducedD = np.zeros((coords.shape[0], coords.shape[0])) - for i in xrange(coords.shape[0]): - for j in xrange(coords.shape[0]): - d = coords[i,:] - coords[j,:] - reducedD[i, j] = np.sqrt( d.dot(d) ) - reducedD = reducedD[np.tril_indices_from(reducedD, k=-1)] - originalD = self.y2[np.tril_indices_from(self.y2, k=-1)] - xmin = np.min(reducedD) - xmax = np.max(reducedD) - f, ax = py.subplots() - ax.plot(reducedD, originalD, 'ko') - ax.plot([xmin, xmax], [xmin, xmax], 'r--') - ax.set_xlabel('Distances in Reduced Space') - ax.set_ylabel('Distances in Original Matrix') - py.show() + """ + def __init__(self, x, correction=None, siteNames=None): + if not isinstance(x, (DataFrame, np.ndarray)): + msg = "Data must either be pandas.DataFrame or nump.ndarray" + raise ValueError(msg) + if isinstance(x, DataFrame): + if (x.dtypes == "object").any(): + msg = "DataFrame can only contain numeric values" + raise ValueError(msg) + y = np.array(x) + if isinstance(x, np.ndarray): + y = x + if np.isnan(y).any(): + msg = "Distance matrix contains null values" + raise ValueError(msg) + if y.any() < 0: + msg = "Distance matrix cannot contain negative values" + raise ValueError(msg) + if y.shape[0] != y.shape[1]: + msg = "Distance matrix must be square" + raise ValueError(msg) + if not np.allclose(y.T, y): + msg = "Distance matrix must be symmetric" + raise ValueError(msg) + A = -0.5 * np.square(y.astype("float")) + n = y.shape[0] + ones = np.ones(n)[np.newaxis].T + I = np.eye(n) + D = (I - ones.dot(ones.T) / n).dot(A).dot(I - ones.dot(ones.T) / n) + self.evals, self.U = np.linalg.eig(D) + self.evals = self.evals.real + self.U = self.U.real + idx = self.evals.argsort()[::-1] + self.U = self.U[:, idx] + if correction is not None: + if correction not in ["1", "2"]: + msg = "correction must be either '1' or '2'" + raise ValueError(msg) + if correction is "1": + negEvl = np.abs(np.min(self.evals[self.evals < 0])) + A = -0.5 * np.square(y) - negEvl + np.fill_diagonal(A, 0) + D = (I - ones.dot(ones.T) / n).dot(A).dot(I - ones.dot(ones.T) / n) + self.evals, self.U = np.linalg.eig(D) + idx = self.evals.argsort()[::-1] + self.U = self.U[:, idx] + self.correction = negEvl + if correction is "2": + mat0 = np.zeros((n, n)) + matI = -1.0 * np.eye(n) + d1 = 2.0 * D + d2 = -0.5 * y + d2 = -4.0 * (I - ones.dot(ones.T) / n).dot(d2).dot(I - ones.dot(ones.T) / n) + t1 = np.concatenate((mat0, matI), axis=0) + t2 = np.concatenate((d1, d2), axis=0) + specMat = np.concatenate((t1, t2), axis=1) + t_evals, t_evecs = np.linalg.eig(specMat) + posEvl = np.max(np.real(t_evals)) + A = -0.5 * (np.square(y.astype("float") + posEvl)) + np.fill_diagonal(A, 0) + D = (I - ones.dot(ones.T) / n).dot(A).dot(I - ones.dot(ones.T) / n) + self.evals, self.U = np.linalg.eig(D) + idx = self.evals.argsort()[::-1] + self.U = self.U[:, idx] + self.correction = posEvl + self.evals = np.round(self.evals[idx], 4) + self.U = np.round(self.U.dot(np.diag(np.sqrt(self.evals))), 4) + self.siteLabs = ["Site " + str(x) for x in range(1, y.shape[0] + 1)] + if isinstance(x, DataFrame): + self.siteLabs = x.index + if siteNames is not None: + self.siteLabs = siteNames + self.y2 = y + def summary(self): + sds = np.sqrt(self.evals) + props = self.evals / np.sum(self.evals) + cumSums = np.cumsum(self.evals) / np.sum(self.evals) + colNames = ["PCoA Axis " + str(x) for x in range(1, len(self.evals) + 1)] + sumTable = DataFrame( + np.vstack((sds, props, cumSums)), index=["Std. Dev", "Prop.", "Cum. Prop."] + ) + sumTable.columns = colNames + return sumTable + def biplot( + self, + coords=False, + xax=1, + yax=2, + descriptors=None, + descripNames=None, + spCol="r", + siteCol="k", + spSize=12, + siteSize=12, + ): + if descriptors is not None: + warn( + "\nWarning: Descriptors must not be binary.\nIgnore this message if all descriptors are quantitative\n" + ) + if not isinstance(descriptors, (DataFrame, np.ndarray)): + msg = "descriptors must be a pandas.DataFrame or numpy.ndarray" + raise ValueError(msg) + if isinstance(descriptors, DataFrame): + if (descriptors.dtypes == "object").any(): + msg = "DataFrame can only contain numeric values" + raise ValueError(msg) + d2 = np.array(descriptors) + d2 = np.apply_along_axis( + lambda x: (x - np.mean(x)) / np.std(x, ddof=1), 0, d2 + ) + if isinstance(descriptors, DataFrame): + dLabs = descriptors.columns.values + elif descripNames is not None: + if len(descripNames) != d2.shape[1]: + msg = "descripNames must be equal to the number of columns in descriptors" + raise ValueError(msg) + dLabs = descripNames + else: + dLabs = ["D" + str(x) for x in range(1, d2.shape[1] + 1)] + U2 = self.U[:, [xax - 1, yax - 1]].astype("float") + U2 = np.apply_along_axis( + lambda x: (x - np.mean(x)) / np.std(x, ddof=1), 0, U2 + ) + S = (1.0 / (d2.shape[0] - 1)) * d2.T.dot(U2) + dProj = np.sqrt(d2.shape[0] - 1) * S.dot( + np.diag(self.evals[[xax - 1, yax - 2]] ** -0.5) + ) + if not coords: + f, ax = py.subplots() + ax.axvline(0, ls="solid", c="k") + ax.axhline(0, ls="solid", c="k") + ax.plot(self.U[:, xax - 1], self.U[:, yax - 1], "ko", ms=0) + [ + ax.text( + x, y, s, color=siteCol, fontsize=siteSize, ha="center", va="center" + ) + for x, y, s in zip( + self.U[:, xax - 1], self.U[:, yax - 1], self.siteLabs + ) + ] + if descriptors is not None: + ax.plot(dProj[:, 0], dProj[:, 1], "ko", ms=0) + [ + ax.text( + x, y, s, color=spCol, fontsize=spSize, ha="center", va="center" + ) + for x, y, s in zip(dProj[:, 0], dProj[:, 1], dLabs) + ] + ax.set_xlabel("PCoA Axis {!s}".format(xax)) + ax.set_ylabel("PCoA Axis {!s}".format(yax)) + py.show() + else: + if descriptors is not None: + return {"Objects": self.U[:, [xax - 1, yax - 1]], "Descriptors": dProj} + else: + return {"Objects": self.U[:, [xax - 1, yax - 1]]} - \ No newline at end of file + def shepard(self, xax=1, yax=2): + coords = self.U[:, [xax - 1, yax - 1]] + reducedD = np.zeros((coords.shape[0], coords.shape[0])) + for i in xrange(coords.shape[0]): + for j in xrange(coords.shape[0]): + d = coords[i, :] - coords[j, :] + reducedD[i, j] = np.sqrt(d.dot(d)) + reducedD = reducedD[np.tril_indices_from(reducedD, k=-1)] + originalD = self.y2[np.tril_indices_from(self.y2, k=-1)] + xmin = np.min(reducedD) + xmax = np.max(reducedD) + f, ax = py.subplots() + ax.plot(reducedD, originalD, "ko") + ax.plot([xmin, xmax], [xmin, xmax], "r--") + ax.set_xlabel("Distances in Reduced Space") + ax.set_ylabel("Distances in Original Matrix") + py.show() diff --git a/ecopy/ordination/transform.py b/ecopy/ordination/transform.py index 0264a76..7aff28b 100644 --- a/ecopy/ordination/transform.py +++ b/ecopy/ordination/transform.py @@ -1,8 +1,9 @@ import numpy as np -from pandas import DataFrame +from pandas import DataFrame -def transform(x, method='wisconsin', axis=1, breakNA=True): - ''' + +def transform(x, method="wisconsin", axis=1, breakNA=True): + """ Docstring for function ecopy.distance ======================== Applies a transformation to a given matrix @@ -42,125 +43,141 @@ def transform(x, method='wisconsin', axis=1, breakNA=True): # divide each element by row total ep.transform(varespec, method='total', axis=1) - ''' - if not isinstance(breakNA, bool): - msg = 'breakNA must be boolean' - raise ValueError(msg) - if not isinstance(x, (DataFrame, np.ndarray)): - msg = 'x must be either numpy array or dataframe' - raise ValueError(msg) - if axis not in [0, 1]: - msg = 'Axis argument must be either 0 or 1' - raise ValueError(msg) - if method not in ['total', 'max', 'normalize', 'range', 'standardize', 'hellinger', 'log', 'logp1', 'pa', 'wisconsin']: - msg = '{0} not an accepted method'.format(method) - raise ValueError(msg) - if isinstance(x, DataFrame): - if breakNA: - if x.isnull().any().any(): - msg = 'DataFrame contains null values' - raise ValueError(msg) - if (x<0).any().any(): - msg = 'DataFrame contains negative values' - raise ValueError(msg) - z = x.copy() - if method=='total': - data = z.apply(totalTrans, axis=axis) - return data - if method=='max': - data = z.apply(maxTrans, axis=axis) - return data - if method=='normalize': - data = z.apply(normTrans, axis=axis) - return data - if method=='range': - data = z.apply(rangeTrans, axis=axis) - return data - if method=='standardize': - data = z.apply(standTrans, axis=axis) - return data - if method=='pa': - z[z>0] = 1 - return z - if method=='hellinger': - data = z.apply(totalTrans, axis=axis) - return np.sqrt(data) - if method=='log': - if ((z > 0) & (z < 1)).any().any(): - msg = 'Log of values between 0 and 1 will return negative numbers\nwhich cannot be used in subsequent distance calculations' - raise ValueError(msg) - data = z.applymap(lambda y: np.log(y+1)) - return data - if method=='logp1': - if ((z > 0) & (z < 1)).any().any(): - msg = 'Log of values between 0 and 1 will return negative numbers\nwhich cannot be used in subsequent distance calculations' - raise ValueError(msg) - data = z.applymap(lambda y: np.log(y) + 1 if y>0 else 0) - return data - if method=='wisconsin': - data = z.apply(maxTrans, axis=0) - data = data.apply(totalTrans, axis=1) - return data - if isinstance(x, np.ndarray): - if breakNA: - if np.isnan(np.sum(x)): - msg = 'Array contains null values' - raise ValueError(msg) - if (x < 0).any(): - msg = 'Array contains negative values' - raise ValueError(msg) - z = x.copy() - if method=='total': - data = np.apply_along_axis(totalTrans, axis, z) - return data - if method=='max': - data = np.apply_along_axis(maxTrans, axis, z) - return data - if method=='normalize': - data = np.apply_along_axis(normTrans, axis, z) - return data - if method=='range': - data = np.apply_along_axis(rangeTrans, axis, z) - return data - if method=='standardize': - data = np.apply_along_axis(standTrans, axis, z) - return data - if method=='pa': - z[z>0] = 1 - return z - if method=='hellinger': - data = np.apply_along_axis(totalTrans, axis, z) - return np.sqrt(data) - if method=='log': - if ((z > 0) & (z < 1)).any(): - msg = 'Log of values between 0 and 1 will return negative numbers\nwhich cannot be used in subsequent distance calculations' - raise ValueError(msg) - data = np.log(z + 1) - return data - if method=='logp1': - if ((z > 0) & (z < 1)).any(): - msg = 'Log of values between 0 and 1 will return negative numbers\nwhich cannot be used in subsequent distance calculations' - raise ValueError(msg) - data = z.astype('float') - data[np.greater(data,0)] = np.log(data[np.greater(data,0)]) + 1 - return data - if method=='wisconsin': - data = np.apply_along_axis(maxTrans, 0, z) - data = np.apply_along_axis(totalTrans, 1, z) - return data + """ + if not isinstance(breakNA, bool): + msg = "breakNA must be boolean" + raise ValueError(msg) + if not isinstance(x, (DataFrame, np.ndarray)): + msg = "x must be either numpy array or dataframe" + raise ValueError(msg) + if axis not in [0, 1]: + msg = "Axis argument must be either 0 or 1" + raise ValueError(msg) + if method not in [ + "total", + "max", + "normalize", + "range", + "standardize", + "hellinger", + "log", + "logp1", + "pa", + "wisconsin", + ]: + msg = "{0} not an accepted method".format(method) + raise ValueError(msg) + if isinstance(x, DataFrame): + if breakNA: + if x.isnull().any().any(): + msg = "DataFrame contains null values" + raise ValueError(msg) + if (x < 0).any().any(): + msg = "DataFrame contains negative values" + raise ValueError(msg) + z = x.copy() + if method == "total": + data = z.apply(totalTrans, axis=axis) + return data + if method == "max": + data = z.apply(maxTrans, axis=axis) + return data + if method == "normalize": + data = z.apply(normTrans, axis=axis) + return data + if method == "range": + data = z.apply(rangeTrans, axis=axis) + return data + if method == "standardize": + data = z.apply(standTrans, axis=axis) + return data + if method == "pa": + z[z > 0] = 1 + return z + if method == "hellinger": + data = z.apply(totalTrans, axis=axis) + return np.sqrt(data) + if method == "log": + if ((z > 0) & (z < 1)).any().any(): + msg = "Log of values between 0 and 1 will return negative numbers\nwhich cannot be used in subsequent distance calculations" + raise ValueError(msg) + data = z.applymap(lambda y: np.log(y + 1)) + return data + if method == "logp1": + if ((z > 0) & (z < 1)).any().any(): + msg = "Log of values between 0 and 1 will return negative numbers\nwhich cannot be used in subsequent distance calculations" + raise ValueError(msg) + data = z.applymap(lambda y: np.log(y) + 1 if y > 0 else 0) + return data + if method == "wisconsin": + data = z.apply(maxTrans, axis=0) + data = data.apply(totalTrans, axis=1) + return data + if isinstance(x, np.ndarray): + if breakNA: + if np.isnan(np.sum(x)): + msg = "Array contains null values" + raise ValueError(msg) + if (x < 0).any(): + msg = "Array contains negative values" + raise ValueError(msg) + z = x.copy() + if method == "total": + data = np.apply_along_axis(totalTrans, axis, z) + return data + if method == "max": + data = np.apply_along_axis(maxTrans, axis, z) + return data + if method == "normalize": + data = np.apply_along_axis(normTrans, axis, z) + return data + if method == "range": + data = np.apply_along_axis(rangeTrans, axis, z) + return data + if method == "standardize": + data = np.apply_along_axis(standTrans, axis, z) + return data + if method == "pa": + z[z > 0] = 1 + return z + if method == "hellinger": + data = np.apply_along_axis(totalTrans, axis, z) + return np.sqrt(data) + if method == "log": + if ((z > 0) & (z < 1)).any(): + msg = "Log of values between 0 and 1 will return negative numbers\nwhich cannot be used in subsequent distance calculations" + raise ValueError(msg) + data = np.log(z + 1) + return data + if method == "logp1": + if ((z > 0) & (z < 1)).any(): + msg = "Log of values between 0 and 1 will return negative numbers\nwhich cannot be used in subsequent distance calculations" + raise ValueError(msg) + data = z.astype("float") + data[np.greater(data, 0)] = np.log(data[np.greater(data, 0)]) + 1 + return data + if method == "wisconsin": + data = np.apply_along_axis(maxTrans, 0, z) + data = np.apply_along_axis(totalTrans, 1, z) + return data + def totalTrans(y): - return y/np.nansum(y) + return y / np.nansum(y) + def maxTrans(y): - return y/np.nanmax(y) + return y / np.nanmax(y) + def normTrans(y): - denom = np.sqrt(np.nansum(y**2)) - return y/denom + denom = np.sqrt(np.nansum(y ** 2)) + return y / denom + def rangeTrans(y): - return (y - np.nanmin(y))/(np.nanmax(y) - np.nanmin(y)) + return (y - np.nanmin(y)) / (np.nanmax(y) - np.nanmin(y)) + def standTrans(y): - return (y - np.nanmean(y))/np.nanstd(y, ddof=1) + return (y - np.nanmean(y)) / np.nanstd(y, ddof=1) diff --git a/ecopy/regression/__init__.py b/ecopy/regression/__init__.py index 88b0ed2..960a2c0 100644 --- a/ecopy/regression/__init__.py +++ b/ecopy/regression/__init__.py @@ -1,2 +1,2 @@ from .nls import nls -from .isoregress import isotonic \ No newline at end of file +from .isoregress import isotonic diff --git a/ecopy/regression/isoregress.py b/ecopy/regression/isoregress.py index ce604bc..5f1dada 100644 --- a/ecopy/regression/isoregress.py +++ b/ecopy/regression/isoregress.py @@ -3,8 +3,9 @@ import matplotlib.pyplot as plt from .isoFunc import _isotonic_regression + class isotonic: - ''' + """ Docstring for function ecopy.isotonic ============================= @@ -44,53 +45,58 @@ class isotonic: pit = np.array([21.0, 23.5, 23.0, 24.0, 21.0, 25.0, 21.5, 22.0, 19.0, 23.5, 25.0]) solution = ep.isotonic(pit, age) solution.plot() - ''' - def __init__(self, y, x=None, w=None, direction='increasing'): - if w is None: - w = np.ones(len(y)) - if x is None: - x = np.arange(len(y)) - if not isinstance(y, (DataFrame, Series, np.ndarray, list)): - msg = 'Response variable (y) must be a pandas.DataFrame, pandas.Series, numpy.ndarray, or list' - raise ValueError(msg) - if not isinstance(x, (DataFrame, Series, np.ndarray, list)): - msg = 'Predictor variable (x) must be a pandas.DataFrame, pandas.Series, numpy.ndarray, or list' - raise ValueError(msg) - if not isinstance(w, (DataFrame, Series, np.ndarray, list)): - msg = 'Weights vector (w) must be a pandas.DataFrame, pandas.Series, numpy.ndarray, or list' - raise ValueError(msg) - if direction not in ['increasing', 'decreasing']: - msg = "direction must be either 'increasing' or 'decreasing'" - raise ValueError(msg) - x = np.array(x) - y = np.array(y) - w = np.array(w) - o = x.argsort() # sorting - o2 = np.zeros(len(x), dtype='int') - for i in range(len(y)): - o2[o[i]] = i # get the original order (i.e. return the second sorted value to its original position) - self.predictor = x - y = y[o] - w = w[o] - if direction is 'decreasing': - y = y[::-1] # flip y around and use the same algorithm - self.prediction = _isotonic_regression(y, w, np.ones(len(y))) - self.prediction = self.prediction[o2] # put into original order - self.obs = y[o2] + """ + + def __init__(self, y, x=None, w=None, direction="increasing"): + if w is None: + w = np.ones(len(y)) + if x is None: + x = np.arange(len(y)) + if not isinstance(y, (DataFrame, Series, np.ndarray, list)): + msg = "Response variable (y) must be a pandas.DataFrame, pandas.Series, numpy.ndarray, or list" + raise ValueError(msg) + if not isinstance(x, (DataFrame, Series, np.ndarray, list)): + msg = "Predictor variable (x) must be a pandas.DataFrame, pandas.Series, numpy.ndarray, or list" + raise ValueError(msg) + if not isinstance(w, (DataFrame, Series, np.ndarray, list)): + msg = "Weights vector (w) must be a pandas.DataFrame, pandas.Series, numpy.ndarray, or list" + raise ValueError(msg) + if direction not in ["increasing", "decreasing"]: + msg = "direction must be either 'increasing' or 'decreasing'" + raise ValueError(msg) + x = np.array(x) + y = np.array(y) + w = np.array(w) + o = x.argsort() # sorting + o2 = np.zeros(len(x), dtype="int") + for i in range(len(y)): + o2[ + o[i] + ] = ( + i + ) # get the original order (i.e. return the second sorted value to its original position) + self.predictor = x + y = y[o] + w = w[o] + if direction is "decreasing": + y = y[::-1] # flip y around and use the same algorithm + self.prediction = _isotonic_regression(y, w, np.ones(len(y))) + self.prediction = self.prediction[o2] # put into original order + self.obs = y[o2] - def plot(self): - o = self.predictor.argsort() - xplt = self.predictor[o] - yplt = self.obs[o] - predplt = self.prediction[o] - f, ax = plt.subplots() - ax.scatter(xplt, yplt, c='r', s=50, label='Original Data') - ax.step(xplt, predplt, label='Prediction') - ax.set_ylabel("Response") - ax.set_xlabel("Predictor") - ax.spines['top'].set_visible(False) - ax.spines['right'].set_visible(False) - ax.yaxis.set_ticks_position('left') - ax.xaxis.set_ticks_position('bottom') - ax.legend(loc='best') - plt.show() + def plot(self): + o = self.predictor.argsort() + xplt = self.predictor[o] + yplt = self.obs[o] + predplt = self.prediction[o] + f, ax = plt.subplots() + ax.scatter(xplt, yplt, c="r", s=50, label="Original Data") + ax.step(xplt, predplt, label="Prediction") + ax.set_ylabel("Response") + ax.set_xlabel("Predictor") + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + ax.yaxis.set_ticks_position("left") + ax.xaxis.set_ticks_position("bottom") + ax.legend(loc="best") + plt.show() diff --git a/ecopy/regression/nls.py b/ecopy/regression/nls.py index 514468a..2c57fe3 100644 --- a/ecopy/regression/nls.py +++ b/ecopy/regression/nls.py @@ -1,5 +1,5 @@ class nls(object): - ''' + """ Docstring for function ecopy.nls ================= Provides a wrapper for scipy.optimize.leastsq. The function should @@ -65,70 +65,88 @@ def tempMod(params, X, Y): tMod = ep.nls(tempMod, p0, X, Y) tMod.summary() tMod.AIC() - ''' - def __init__(self, func, p0, xdata, ydata): - import numpy as np - from scipy.optimize import leastsq - import scipy.stats as spst - # Check the data - if len(xdata) != len(ydata): - msg = 'The number of observations does not match the number of rows for the predictors' - raise ValueError(msg) - # Check for NA's in the predictor or response - if np.isnan(xdata).any() == True: - msg = 'Predicor variable (x) contains missing values' - raise ValueError(msg) - if np.isnan(ydata).any() == True: - msg = 'Response variable (y) contains missing values' - raise ValueError(msg) - # Check parameter estimates - if type(p0) != dict: - msg = "Initial parameter estimates (p0) must be a dictionary of form p0={'a':1, 'b':2, etc}" - raise ValueError(msg) - self.func = func - self.inits = p0.values() - self.nobs = len(ydata) - self.nparm= len(self.inits) - self.parmNames = p0.keys() - for i in range(len(self.parmNames)): - if len(self.parmNames[i]) > 5: - self.parmNames[i] = self.parmNames[i][0:4] - # Run the model - self.mod1 = leastsq(self.func, self.inits, args = (xdata, ydata), full_output=1) - # Get the parameters - self.parmEsts = np.round(self.mod1[0], 4) - # Get the Error variance and standard deviation - self.RSS = np.sum(self.mod1[2]['fvec']**2) - self.df = self.nobs - self.nparm - self.MSE = self.RSS / self.df - self.RMSE = np.sqrt(self.MSE) - # Get the covariance matrix - self.cov = self.MSE * self.mod1[1] - # Get parameter standard errors - self.parmSE = np.diag(np.sqrt(self.cov)) - # Calculate the t-values - self.tvals = self.parmEsts/self.parmSE - # Get p-values - self.pvals = (1 - spst.t.cdf(np.abs(self.tvals), self.df))*2 - # Get biased variance (MLE) and calculate log-likehood - self.s2b = self.RSS / self.nobs - self.logLik = -self.nobs/2. * np.log(2.*np.pi) - self.nobs/2. * np.log(self.s2b) - 1./(2.*self.s2b) * self.RSS - del(self.mod1) - del(self.s2b) - - # Get AIC. Add 1 to the df to account for estimation of standard error - def AIC(self, k=2): - print('AIC: {0:5.4f}'.format(-2*self.logLik + k*(self.nparm + 1))) + """ - # Print the summary - def summary(self): - print('') - print('Non-linear least squares') - print('Model: ' + self.func.func_name) - print('Parameters:') - print('{0:^5} {1:^8} {2:^5} {3:^8} {4:^8}'.format(' ','Estimate', 'Std. Error', 't-value', 'P(>|t|)')) - for i in range( len(self.parmNames) ): - print('{0:<5} {1:^8} {2:^8.4f} {3:^8.4} {4:^8.4}'.format(self.parmNames[i], self.parmEsts[i], self.parmSE[i], self.tvals[i], self.pvals[i])) - print('') - print('Residual Standard Error: {0:5.4f}'.format(self.RMSE)) - print('Df: {0}'.format(self.df)) + def __init__(self, func, p0, xdata, ydata): + import numpy as np + from scipy.optimize import leastsq + import scipy.stats as spst + + # Check the data + if len(xdata) != len(ydata): + msg = "The number of observations does not match the number of rows for the predictors" + raise ValueError(msg) + # Check for NA's in the predictor or response + if np.isnan(xdata).any() == True: + msg = "Predicor variable (x) contains missing values" + raise ValueError(msg) + if np.isnan(ydata).any() == True: + msg = "Response variable (y) contains missing values" + raise ValueError(msg) + # Check parameter estimates + if type(p0) != dict: + msg = "Initial parameter estimates (p0) must be a dictionary of form p0={'a':1, 'b':2, etc}" + raise ValueError(msg) + self.func = func + self.inits = p0.values() + self.nobs = len(ydata) + self.nparm = len(self.inits) + self.parmNames = p0.keys() + for i in range(len(self.parmNames)): + if len(self.parmNames[i]) > 5: + self.parmNames[i] = self.parmNames[i][0:4] + # Run the model + self.mod1 = leastsq(self.func, self.inits, args=(xdata, ydata), full_output=1) + # Get the parameters + self.parmEsts = np.round(self.mod1[0], 4) + # Get the Error variance and standard deviation + self.RSS = np.sum(self.mod1[2]["fvec"] ** 2) + self.df = self.nobs - self.nparm + self.MSE = self.RSS / self.df + self.RMSE = np.sqrt(self.MSE) + # Get the covariance matrix + self.cov = self.MSE * self.mod1[1] + # Get parameter standard errors + self.parmSE = np.diag(np.sqrt(self.cov)) + # Calculate the t-values + self.tvals = self.parmEsts / self.parmSE + # Get p-values + self.pvals = (1 - spst.t.cdf(np.abs(self.tvals), self.df)) * 2 + # Get biased variance (MLE) and calculate log-likehood + self.s2b = self.RSS / self.nobs + self.logLik = ( + -self.nobs / 2.0 * np.log(2.0 * np.pi) + - self.nobs / 2.0 * np.log(self.s2b) + - 1.0 / (2.0 * self.s2b) * self.RSS + ) + del self.mod1 + del self.s2b + + # Get AIC. Add 1 to the df to account for estimation of standard error + def AIC(self, k=2): + print("AIC: {0:5.4f}".format(-2 * self.logLik + k * (self.nparm + 1))) + + # Print the summary + def summary(self): + print("") + print("Non-linear least squares") + print("Model: " + self.func.func_name) + print("Parameters:") + print( + "{0:^5} {1:^8} {2:^5} {3:^8} {4:^8}".format( + " ", "Estimate", "Std. Error", "t-value", "P(>|t|)" + ) + ) + for i in range(len(self.parmNames)): + print( + "{0:<5} {1:^8} {2:^8.4f} {3:^8.4} {4:^8.4}".format( + self.parmNames[i], + self.parmEsts[i], + self.parmSE[i], + self.tvals[i], + self.pvals[i], + ) + ) + print("") + print("Residual Standard Error: {0:5.4f}".format(self.RMSE)) + print("Df: {0}".format(self.df)) diff --git a/ecopy/test_ecopy_unittest.py b/ecopy/test_ecopy_unittest.py index 642f44e..98a0079 100644 --- a/ecopy/test_ecopy_unittest.py +++ b/ecopy/test_ecopy_unittest.py @@ -2,39 +2,40 @@ import numpy as np from ecopy import load_data, wt_mean, wt_var, wt_scale, diversity, rarefy -class TestECOPY(unittest.TestCase): - def setUp(self): - pass - - def test_wt_mean(self): - x = [2, 3, 4, 5] - w = [0.1, 0.1, 0.5, 0.3] - res = wt_mean(x, w) - self.assertEqual(res, 4.0) - - def test_wt_var(self): - x = [2, 3, 4, 5] - w = [0.1, 0.1, 0.5, 0.3] - res = wt_var(x, w) - self.assertEqual(res, 1.25) - - def test_wt_scale(self): - x = [2, 3, 4, 5] - w = [0.1, 0.1, 0.5, 0.3] - res = np.round(wt_scale(x,w), 2) - truth = res == [-1.79, -0.89, 0, 0.89] - self.assertEqual(truth.sum(), 4) - - def test_diversity(self): - sp = np.array([0, 1, 2, 3, 0]).reshape(1,5) - div = np.round(diversity(sp)) - self.assertEqual(div, 1) - - def test_rarefy(self): - BCI = load_data('BCI') - rareRich = np.round(rarefy(BCI, 'rarefy')) - self.assertEqual(rareRich[1], 77) - -if __name__ == '__main__': - unittest.main() \ No newline at end of file +class TestECOPY(unittest.TestCase): + def setUp(self): + pass + + def test_wt_mean(self): + x = [2, 3, 4, 5] + w = [0.1, 0.1, 0.5, 0.3] + res = wt_mean(x, w) + self.assertEqual(res, 4.0) + + def test_wt_var(self): + x = [2, 3, 4, 5] + w = [0.1, 0.1, 0.5, 0.3] + res = wt_var(x, w) + self.assertEqual(res, 1.25) + + def test_wt_scale(self): + x = [2, 3, 4, 5] + w = [0.1, 0.1, 0.5, 0.3] + res = np.round(wt_scale(x, w), 2) + truth = res == [-1.79, -0.89, 0, 0.89] + self.assertEqual(truth.sum(), 4) + + def test_diversity(self): + sp = np.array([0, 1, 2, 3, 0]).reshape(1, 5) + div = np.round(diversity(sp)) + self.assertEqual(div, 1) + + def test_rarefy(self): + BCI = load_data("BCI") + rareRich = np.round(rarefy(BCI, "rarefy")) + self.assertEqual(rareRich[1], 77) + + +if __name__ == "__main__": + unittest.main() diff --git a/ecopy/utils.py b/ecopy/utils.py index 2f97fab..8584e49 100644 --- a/ecopy/utils.py +++ b/ecopy/utils.py @@ -1,8 +1,9 @@ import pandas as pd + def load_data(x): - """Loads data from online repository. Requires internet.""" - path = 'https://github.com/Auerilas/ecopy-data/raw/master/data/{0}.csv' - downloadPath = path.format(x) - loaded = pd.read_csv(downloadPath) - return loaded \ No newline at end of file + """Loads data from online repository. Requires internet.""" + path = "https://github.com/Auerilas/ecopy-data/raw/master/data/{0}.csv" + downloadPath = path.format(x) + loaded = pd.read_csv(downloadPath) + return loaded