ValueError: dimensions ('chain', 'draw', 'y_dim_0') must have the same length as the number of data dimensions, ndim=2

Questions v5
1 
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In [33], line 42
     37 bayes_reg.limit_y = LSL_table.loc[idx, ].iloc[0, 0] 
     39 bayes_reg.fit(df_reg[test_for_reg[0]], 
     40               df_reg[test_for_reg[1]],
     41               Intercept = True)
---> 42 bayes_reg.credible_interval()
     43 bayes_reg.credible_interval_loc([bayes_reg.limit_x])
     44 #bayes_reg.distribution_plot_at_x()

Cell In [32], line 25, in release_test_correlation.credible_interval(self, nb_points)
     23 self.x_data_shared.set_value(x_for_interp)
     24 self.y_data_shared.set_value(np.zeros_like(x_for_interp))
---> 25 self.credible_interval_result, self.sampled_posterior_CI = credible_interval_calculation(self.x_data_shared, 
     26                                                               self.trace_linear, self.linear_model)

Cell In [28], line 10, in credible_interval_calculation(x_data_shared, trace_linear, linear_model)
      8 nb_point = x_data_shared.get_value().shape[0]
      9 print(nb_point)
---> 10 predicted = np.reshape(predicted, (n_samples, nb_point))    
     11 predicted_interval = []
     12 for col in predicted.T:

File <__array_function__ internals>:180, in reshape(*args, **kwargs)

File c:\ProgramData\MiniforgeEnvs\pymc_env\lib\site-packages\numpy\core\fromnumeric.py:298, in reshape(a, newshape, order)
    198 @array_function_dispatch(_reshape_dispatcher)
    199 def reshape(a, newshape, order='C'):
    200     """
    201     Gives a new shape to an array without changing its data.
    202 
   (...)
    296            [5, 6]])
    297     """
--> 298     return _wrapfunc(a, 'reshape', newshape, order=order)

File c:\ProgramData\MiniforgeEnvs\pymc_env\lib\site-packages\numpy\core\fromnumeric.py:54, in _wrapfunc(obj, method, *args, **kwds)
     52 bound = getattr(obj, method, None)
     53 if bound is None:
---> 54     return _wrapit(obj, method, *args, **kwds)
     56 try:
     57     return bound(*args, **kwds)

File c:\ProgramData\MiniforgeEnvs\pymc_env\lib\site-packages\numpy\core\fromnumeric.py:47, in _wrapit(obj, method, *args, **kwds)
     45     if not isinstance(result, mu.ndarray):
     46         result = asarray(result)
---> 47     result = wrap(result)
 48 return result

File c:\ProgramData\MiniforgeEnvs\pymc_env\lib\site-packages\xarray\core\dataarray.py:3487, in DataArray.__array_wrap__(self, obj, context)
   3486 def __array_wrap__(self: T_DataArray, obj, context=None) -> T_DataArray:
-> 3487     new_var = self.variable.__array_wrap__(obj, context)
   3488     return self._replace(new_var)

File c:\ProgramData\MiniforgeEnvs\pymc_env\lib\site-packages\xarray\core\variable.py:2496, in Variable.__array_wrap__(self, obj, context)
   2495 def __array_wrap__(self, obj, context=None):
-> 2496     return Variable(self.dims, obj)

File c:\ProgramData\MiniforgeEnvs\pymc_env\lib\site-packages\xarray\core\variable.py:320, in Variable.__init__(self, dims, data, attrs, encoding, fastpath)
    300 """
    301 Parameters
    302 ----------
   (...)
    317     unrecognized encoding items.
    318 """
    319 self._data = as_compatible_data(data, fastpath=fastpath)
--> 320 self._dims = self._parse_dimensions(dims)
    321 self._attrs = None
    322 self._encoding = None

File c:\ProgramData\MiniforgeEnvs\pymc_env\lib\site-packages\xarray\core\variable.py:580, in Variable._parse_dimensions(self, dims)
    578 dims = tuple(dims)
    579 if len(dims) != self.ndim:
--> 580     raise ValueError(
    581         f"dimensions {dims} must have the same length as the "
    582         f"number of data dimensions, ndim={self.ndim}"
    583     )
    584 return dims

ValueError: dimensions ('chain', 'draw', 'y_dim_0') must have the same length as the number of data dimensions, ndim=2

Code:

def bayesian_lin_reg_fit(x_data_shared,y_data_shared,display_plot=True,Intercept=False):
    
    with pm.Model() as linear_model:
        sigma = pm.Uniform('Sigma',0,100)
        intercept = pm.Uniform('Intercept',0,1200)
        x_coeff = pm.Uniform('Slope',-10,10)
        if(Intercept==True):
            yhat = pm.math.dot(x_data_shared,x_coeff)+intercept
        else:
            yhat = pm.math.dot(x_data_shared,x_coeff)
        #Likelihood
        likelihood = pm.Normal('y',yhat,sigma,observed=y_data_shared)
        print(likelihood)
        

        #Inference
        trace_linear = pm.sample(500,cores=1,init="auto",tune=500,progressbar=True)
        
        if(display_plot):
            plt.figure(figsize=(7,7))
            # traceplot(trace_linear[100:])
            plt.tight_layout()

        print(pm.summary(trace_linear))
        return linear_model,trace_linear
        
def credible_interval_calculation(x_data_shared,trace_linear,linear_model):
    n_samples=1000
    ppc_CI = pm.sample_posterior_predictive(trace_linear,
                                            model=linear_model,
                                            progressbar=True)
    predicted = ppc_CI.posterior_predictive['y']
    print(predicted.shape)
    nb_point = x_data_shared.get_value().shape[0]
    print(nb_point)
    predicted = np.reshape(predicted, (n_samples, nb_point))    
    predicted_interval = []
    for col in predicted.T:
        predicted_loc = [np.quantile(col, x) for x in [0.0005, 0.005, 0.05]] + \
                        [np.mean(col)] + \
                        [np.quantile(col, x) for x in [0.95, 0.995, 0.9995]]
        predicted_interval += predicted_loc
        
    predicted_interval = np.array(predicted_interval)
    predicted_interval = np.reshape(predicted_interval, (int(predicted_interval.size / 7), 7), order='C')
    predicted_interval = pd.DataFrame(predicted_interval, columns=('0.05%', '0.5%', '5.%',
                                                                   'mean',
                                                                   '95.%', '99.5%', '99.95%'))
    print('predicted_interval:', predicted_interval)
    predicted_interval['x'] = x_data_shared.get_value()#x_for_interp

    return (predicted_interval, predicted)

How can i reshape my sample posterior predicted when i have more chains.

2

Please provide a working example, a minimum working example if at all possible. Right now, you code is incomplete and won’t run, which makes it difficult to sort out where you’re problem is.

3

Please find the working code:

import arviz as az
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import pymc as pm
print(pm.__version__)
from aesara import shared

def bayesian_lin_reg_fit(x_data_shared,y_data_shared,display_plot=True,Intercept=True):  
    with pm.Model() as linear_model:
        sigma = pm.Uniform('Sigma',0,100)
        intercept = pm.Uniform('Intercept',0,1200)
        x_coeff = pm.Uniform('Slope',-10,10)
        if(Intercept==True):
            yhat = pm.math.dot(x_data_shared,x_coeff)+intercept
        else:
            yhat = pm.math.dot(x_data_shared,x_coeff)
        #Likelihood
        likelihood = pm.Normal('y',yhat,sigma,observed=y_data_shared)
        print(likelihood)
        

        #Inference
        trace_linear = pm.sample(500,cores=1,init="auto",tune=500,progressbar=True)
        
        if(display_plot):
            plt.figure(figsize=(7,7))
            az.plot_trace(trace_linear,combined=True)
            plt.tight_layout()

        print(pm.summary(trace_linear))
        return linear_model,trace_linear

bayes_reg = bayesian_lin_reg_fit(x_data_shared=x_data_shared,y_data_shared=y_data_shared)
ppc_CI = pm.sample_posterior_predictive(bayes_reg[1],
                                            model=bayes_reg[0],
                                            progressbar=True)
predicted = ppc_CI.posterior_predictive['y']
n_samples = 1000
nb_point = x_data_shared.get_value().shape[0]
x = np.reshape(predicted,(n_samples,nb_point))

Error:

**ValueError** : dimensions ('chain', 'draw', 'y_dim_0') must have the same length as the number of data dimensions, ndim=2
4

I still don’t know what x_data_shared or y_data_shared are. However, it looks like you are trying to manipulate predicted as if it is a numpy array. predicted is an arviz InferenceData object, which is basically an xarray object. If you need to manipulate this object “by hand”, you need to do so as one would do with an xarray object, not a simple numpy array.

5

x_data_sahred and y_data_shared are array of data, which i want to model - y as a function of x using probabilistic model.