Hi!
I am building a time series prediction model using Pymc3. The model was comprehensively tested on Spyder, but when migrated to Jupyter Notebook the following error is raised: OSError: [Errno 12] Cannot allocate memory
Below are the relevant section of the code. Any feedback is welcomed.
#%% Model definition
basic_model = pm.Model()
samplesPerYear = 23 #Number of datapoints per year
sigma_err_prior = 1/0.05 #Estimate of EVI error variance
dummyData = np.zeros([0])
shared_data = T.shared(dummyData, borrow=True) #Sahred data definition - Theano
#%% Model definition
with basic_model:
## prior for 15-day EVI, U(0-1)
x = pm.Uniform("x", 0, 1, shape=samplesPerYear)
x = Tensor.cast(x, 'float64') #required for theano
## prior for observation process, Exp(sigma_err_prior)
sigma_err = pm.Exponential("sigma_err", sigma_err_prior)
@T.compile.ops.as_op(itypes=[Tensor.dvector],otypes=[Tensor.dvector]) #original
def loredo(x = x, profile=True):
L = shared_data.get_value(borrow=True).shape[0]
out = np.zeros(L)
k = 0
while True:
if (k+1)*samplesPerYear < L:
out[k*samplesPerYear:(k+1)*samplesPerYear] = x[0:samplesPerYear]
k = k+1
else:
diff = (k+1)*samplesPerYear - L
out[k*samplesPerYear:(k+1)*samplesPerYear-diff] = x[0:samplesPerYear-diff]
break
return out
loredo.grad = lambda *y: y[0] # Here's the klutz, llama al grad pero es un bug
loredo.shape = shared_data.get_value(borrow=True).shape[0]
## likelihood
observation = pm.Normal("obs", mu=loredo(x), tau=1/sigma_err, observed=shared_data)
Problem occurred during compilation with the command line below:
/home/tele/miniconda2/envs/deforestacion3/bin/x86_64-conda_cos6-linux-gnu-c++ -shared -g -O3 -fno-math-errno -Wno-unused-label -Wno-unused-variable -Wno-write-strings -march=haswell -mmmx -mno-3dnow -msse -msse2 -msse3 -mssse3 -mno-sse4a -mcx16 -msahf -mmovbe -maes -mno-sha -mpclmul -mpopcnt -mabm -mno-lwp -mfma -mno-fma4 -mno-xop -mbmi -mno-sgx -mbmi2 -mno-tbm -mavx -mavx2 -msse4.2 -msse4.1 -mlzcnt -mno-rtm -mno-hle -mrdrnd -mf16c -mfsgsbase -mno-rdseed -mno-prfchw -mno-adx -mfxsr -mxsave -mxsaveopt -mno-avx512f -mno-avx512er -mno-avx512cd -mno-avx512pf -mno-prefetchwt1 -mno-clflushopt -mno-xsavec -mno-xsaves -mno-avx512dq -mno-avx512bw -mno-avx512vl -mno-avx512ifma -mno-avx512vbmi -mno-avx5124fmaps -mno-avx5124vnniw -mno-clwb -mno-mwaitx -mno-clzero -mno-pku -mno-rdpid --param l1-cache-size=32 --param l1-cache-line-size=64 --param l2-cache-size=6144 -mtune=haswell -DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION -m64 -fPIC -I/home/tele/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/numpy/core/include -I/home/tele/miniconda2/envs/deforestacion3/include/python3.6m -I/home/tele/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/gof/c_code -L/home/tele/miniconda2/envs/deforestacion3/lib -fvisibility=hidden -o /home/tele/.theano/compiledir_Linux-4.15--generic-x86_64-with-debian-stretch-sid-x86_64-3.6.4-64/tmp67xjcwqf/m03cc3cdee46c6787a94521175277126c8d483d4e6ac5026e943ac127eec8e49a.so /home/tele/.theano/compiledir_Linux-4.15--generic-x86_64-with-debian-stretch-sid-x86_64-3.6.4-64/tmp67xjcwqf/mod.cpp -lpython3.6m
ERROR (theano.gof.cmodule): [Errno 12] Cannot allocate memory
---------------------------------------------------------------------------
OSError Traceback (most recent call last)
<ipython-input-25-8c3ccbaccfd5> in <module>()
----> 1 procesarBayes(path_modis, deforestacion_chips,imagenes_h12_v11, imagenes_h12_v12, mascara, argumentos)
<ipython-input-24-fceb033adb39> in procesarBayes(path, producto, image_list_h12v11, image_list_h12v12, mascara, argumentos)
90 args = (series_evi[:,k], indice_imagen, indice_fecha_inicial_analisis-1, max_counter, nSamples, nBurn, plot)
91 print(args)
---> 92 fecha = algo_bayes_pixel(args)
93 fechas.append(fecha)
94 print(0)
~/Dropbox/Deforestacion/VERSION_OPERACIONAL/descartable/algoritmos.py in algo_bayes_pixel(args)
139 with basic_model:
140 ## prior for 15-day EVI, U(0-1)
--> 141 x = pm.Uniform("x", 0, 1, shape=samplesPerYear)
142 x = Tensor.cast(x, 'float64') #required for theano
143
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/pymc3/distributions/distribution.py in __new__(cls, name, *args, **kwargs)
39 raise TypeError("observed needs to be data but got: {}".format(type(data)))
40 total_size = kwargs.pop('total_size', None)
---> 41 dist = cls.dist(*args, **kwargs)
42 return model.Var(name, dist, data, total_size)
43 else:
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/pymc3/distributions/distribution.py in dist(cls, *args, **kwargs)
50 def dist(cls, *args, **kwargs):
51 dist = object.__new__(cls)
---> 52 dist.__init__(*args, **kwargs)
53 return dist
54
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/pymc3/distributions/continuous.py in __init__(self, lower, upper, *args, **kwargs)
182 self.lower = lower = tt.as_tensor_variable(floatX(lower))
183 self.upper = upper = tt.as_tensor_variable(floatX(upper))
--> 184 self.mean = (upper + lower) / 2.
185 self.median = self.mean
186
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/tensor/var.py in __add__(self, other)
126 def __add__(self, other):
127 try:
--> 128 return theano.tensor.basic.add(self, other)
129 # We should catch the minimum number of exception here.
130 # Otherwise this will convert error when Theano flags
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/gof/op.py in __call__(self, *inputs, **kwargs)
668 # compute output value once with test inputs to validate graph
669 thunk = node.op.make_thunk(node, storage_map, compute_map,
--> 670 no_recycling=[])
671 thunk.inputs = [storage_map[v] for v in node.inputs]
672 thunk.outputs = [storage_map[v] for v in node.outputs]
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/gof/op.py in make_thunk(self, node, storage_map, compute_map, no_recycling, impl)
953 try:
954 return self.make_c_thunk(node, storage_map, compute_map,
--> 955 no_recycling)
956 except (NotImplementedError, utils.MethodNotDefined):
957 # We requested the c code, so don't catch the error.
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/gof/op.py in make_c_thunk(self, node, storage_map, compute_map, no_recycling)
856 _logger.debug('Trying CLinker.make_thunk')
857 outputs = cl.make_thunk(input_storage=node_input_storage,
--> 858 output_storage=node_output_storage)
859 thunk, node_input_filters, node_output_filters = outputs
860
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/gof/cc.py in make_thunk(self, input_storage, output_storage, storage_map, keep_lock)
1215 cthunk, module, in_storage, out_storage, error_storage = self.__compile__(
1216 input_storage, output_storage, storage_map,
-> 1217 keep_lock=keep_lock)
1218
1219 res = _CThunk(cthunk, init_tasks, tasks, error_storage, module)
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/gof/cc.py in __compile__(self, input_storage, output_storage, storage_map, keep_lock)
1155 output_storage,
1156 storage_map,
-> 1157 keep_lock=keep_lock)
1158 return (thunk,
1159 module,
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/gof/cc.py in cthunk_factory(self, error_storage, in_storage, out_storage, storage_map, keep_lock)
1618 node.op.prepare_node(node, storage_map, None, 'c')
1619 module = get_module_cache().module_from_key(
-> 1620 key=key, lnk=self, keep_lock=keep_lock)
1621
1622 vars = self.inputs + self.outputs + self.orphans
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/gof/cmodule.py in module_from_key(self, key, lnk, keep_lock)
1179 try:
1180 location = dlimport_workdir(self.dirname)
-> 1181 module = lnk.compile_cmodule(location)
1182 name = module.__file__
1183 assert name.startswith(location)
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/gof/cc.py in compile_cmodule(self, location)
1521 lib_dirs=self.lib_dirs(),
1522 libs=libs,
-> 1523 preargs=preargs)
1524 except Exception as e:
1525 e.args += (str(self.fgraph),)
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/gof/cmodule.py in compile_str(module_name, src_code, location, include_dirs, lib_dirs, libs, preargs, py_module, hide_symbols)
2344
2345 try:
-> 2346 p_out = output_subprocess_Popen(cmd)
2347 compile_stderr = decode(p_out[1])
2348 except Exception:
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/misc/windows.py in output_subprocess_Popen(command, **params)
75 params['stdout'] = subprocess.PIPE
76 params['stderr'] = subprocess.PIPE
---> 77 p = subprocess_Popen(command, **params)
78 # we need to use communicate to make sure we don't deadlock around
79 # the stdout/stderr pipe.
~/miniconda2/envs/deforestacion3/lib/python3.6/site-packages/theano/misc/windows.py in subprocess_Popen(command, **params)
41
42 try:
---> 43 proc = subprocess.Popen(command, startupinfo=startupinfo, **params)
44 finally:
45 if stdin is not None:
~/miniconda2/envs/deforestacion3/lib/python3.6/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors)
707 c2pread, c2pwrite,
708 errread, errwrite,
--> 709 restore_signals, start_new_session)
710 except:
711 # Cleanup if the child failed starting.
~/miniconda2/envs/deforestacion3/lib/python3.6/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
1273 errread, errwrite,
1274 errpipe_read, errpipe_write,
-> 1275 restore_signals, start_new_session, preexec_fn)
1276 self._child_created = True
1277 finally:
OSError: [Errno 12] Cannot allocate memory