COBAYA: R-1 not improving

[Ali Rida Khalife]

Hello,
I'm running the following python file with version 3.4.1 of COBAYA, and version 3.2.1 of CLASS:

Code: Select all

info={
  "params": {
    "clamp": {
      "latex": "10^9 A_\\mathrm{s} e^{-2\\tau}", 
      "derived": "lambda A_s, tau_reio: 1e9*A_s*np.exp(-2*tau_reio)"
    }, 
    "A": {
      "latex": "10^9 A_\\mathrm{s}", 
      "derived": "lambda A_s: 1e9*A_s"
    }, 
    "Omega_m": {
      "latex": "\\Omega_\\mathrm{m}"
    }, 
    "Omega_Lambda": {
      "latex": "\\Omega_\\Lambda"
    }, 
    "tau_reio": {
        "prior": {"min": 0.01, "max": 0.8}, 
      "proposal": 0.003, 
      "ref": {
        "loc": 0.055, 
        "scale": 0.006, 
        "dist": "norm"
      }, 
      "latex": "\\tau_\\mathrm{reio}"
    }, 
    "age": {
      "latex": "{\\rm{Age}}/\\mathrm{Gyr}"
    }, 
    "n_s": {
      "prior": {
        "max": 1.2, 
        "min": 0.8
      }, 
      "proposal": 0.002, 
      "ref": {
        "loc": 0.965, 
        "scale": 0.004, 
        "dist": "norm"
      }, 
      "latex": "n_\\mathrm{s}"
    }, 
    "s8h5": {
      "latex": "\\sigma_8/h^{0.5}", 
      "derived": "lambda sigma8, H0: sigma8*(H0*1e-2)**(-0.5)"
    },
############### IDM PARAMS BEGIN ###############################
    "m_idm": {
      "latex": "m_\\mathrm{idm}",
      "value": 2e9
    },
    "xi_idr":{
      "prior": {
        "max": 0.5,
        "min": 0
      },
      "proposal": 0.05,
      "ref": {
        "loc": 0.1,
        "scale": 0.05,
        "dist": "norm"
      },
      "latex": r"\xi"
    },
    "log10_a_idm_dr_p1":{
        "prior":{
            "min": 0,
            "max": 8
            },
        "ref":{"dist": "norm","loc": 4, "scale": 3},
        "proposal": 1.5,
        "drop": True,
        "latex": r"\log_{10}(\mathrm{a}_\mathrm{dark}\mathrm{Mpc})"
    },
    "a_idm_dr": {
        "value": "lambda log10_a_idm_dr_p1: 10**(log10_a_idm_dr_p1)-1.0",
        "latex": "\mathrm{a}_\mathrm{dark}"
    },
################ IDM PARAMS FINISH ########################
    "sigma8": {
      "latex": "\\sigma_8"
    }, 
    "z_reio": {
      "latex": "z_\\mathrm{re}"
    }, 
    "omega_b": {
      "prior": {
        "max": 0.1, 
        "min": 0.005
      }, 
      "proposal": 0.0001, 
      "ref": {
        "loc": 0.0224, 
        "scale": 0.0001, 
        "dist": "norm"
      }, 
      "latex": "\\Omega_\\mathrm{b} h^2"
    }, 
    "rs_drag": {
      "latex": "r_\\mathrm{drag}"
    }, 
    "omega_cdm": {
      "prior": {
        "max": 0.99, 
        "min": 0.001
      }, 
      "proposal": 0.0005, 
      "ref": {
        "loc": 0.12, 
        "scale": 0.001, 
        "dist": "norm"
      }, 
      "latex": "\\Omega_\\mathrm{c} h^2"
    }, 
    "s8omegamp25": {
      "latex": "\\sigma_8 \\Omega_\\mathrm{m}^{0.25}", 
      "derived": "lambda sigma8, Omega_m: sigma8*Omega_m**0.25"
    }, 
    "theta_s_1e2": {
      "prior": {
        "max": 10, 
        "min": 0.5
      }, 
      "proposal": 0.0002, 
      "drop": True, 
      "ref": {
        "loc": 1.0416, 
        "scale": 0.0004, 
        "dist": "norm"
      }, 
      "latex": "100\\theta_\\mathrm{s}"
    }, 
    "H0": {
      "latex": "H_0"
    }, 
    "m_ncdm": {
      "value": 0.02, 
      "renames": "mnu"
    },
    "m_ncdm_tot":{
       "latex": r'\Sigma\mathrm{m}_{\nu}\mathrm(eV)'
    },
    "YHe": {
      "latex": "Y_\\mathrm{P}"
    }, 
    "s8omegamp5": {
      "latex": "\\sigma_8 \\Omega_\\mathrm{m}^{0.5}", 
      "derived": "lambda sigma8, Omega_m: sigma8*Omega_m**0.5"
    }, 
    "omegamh2": {
      "latex": "\\Omega_\\mathrm{m} h^2", 
      "derived": "lambda Omega_m, H0: Omega_m*(H0/100)**2"
    }, 
    "100*theta_s": {
      "derived": False, 
      "value": "lambda theta_s_1e2: theta_s_1e2"
    }, 
    "logA": {
      "prior": {
        "max": 3.91, 
        "min": 1.61
      }, 
      "proposal": 0.001, 
      "drop": True, 
      "ref": {
        "loc": 3.05, 
        "scale": 0.001, 
        "dist": "norm"
      }, 
      "latex": "\\log(10^{10} A_\\mathrm{s})"
    }, 
    "A_s": {
      "latex": "A_\\mathrm{s}", 
      "value": "lambda logA: 1e-10*np.exp(logA)"
    }
  }, 
  "theory": {
    "classy": {
      "extra_args": {
        "nindex_idm_dr": 4.0,
        "stat_f_idr": 0.875, #1
        "idr_nature": "free_streaming", # "fluid" 
        ######## PRECISION PARAMS ##########
        #"idm_dr_tight_coupling_trigger_tau_c_over_tau_k":, # when to switch off the dark-tight-coupling approximation, first condition 
        #"idm_dr_tight_coupling_trigger_tau_c_over_tau_h":, # when to switch off the dark-tight-coupling approximation, second condition
        "output": "tCl,pCl,lCl", 
        "f_idm": 1,
        "N_ncdm": 1, 
        "N_ur": 0.00441, 
        "nonlinear_min_k_max": 20, 
        "non linear": "halofit", 
        "deg_ncdm": 3,
      },
      "path": "home/Codes/code/classy/",
      "ignore_obsolete": True

    }
  }, 
  "sampler": {
    "mcmc": {
      "Rminus1_cl_stop": 0.2, 
      "drag": True, 
      "Rminus1_stop": 0.02, 
      "covmat": 'home/IDM/DM_DR/ALL_005/CLASS.covmat', 
      "measure_speeds": True, 
      "oversample_power": 0.4, 
      "output_every": 1, 
      "proposal_scale": 1,
      "max_tries": 1.e4
    }
  }, 
  "resume": True,
  "debug": True,
  "timing": True, 
  "output": "home/IDM/DM_DR/ALL_1/CLASS", 
  "likelihood": {
    "SPT3G_Y1.TTTEEE": {
      "python_path": "home/spt3g"
    }, 
    "planck_2018_lowl.TT": {}, 
    "planck_2018_lowl.EE": {},
    "planck_2018_lensing.clik": {}, 
    "planck_2018_highl_plik.TTTEEE": {}, 
    "sn.pantheon": {}, 
    "bao.sixdf_2011_bao": {}, 
    "bao.sdss_dr7_mgs": {},
    "bao.sdss_dr16_baoplus_lrg": {},
    "bao.sdss_dr16_lrg_bao_dmdh": {},
    #"des_y1.joint": {}
  }
}

import warnings
from sklearn.exceptions import DataConversionWarning
warnings.filterwarnings(action='ignore', category=DataConversionWarning)
from sklearn.exceptions import ConvergenceWarning
warnings.filterwarnings(action='ignore', category=ConvergenceWarning)

from cobaya.run import run

import cobaya

updated_info, sampler = run(info)

At the beginning, the chains were running smoothly towards convergence. Checking the .progress file, I could see that R-1 was being updated every ~4-8 hrs. However, since 10 days ago, R-1 has been stuck and it's not being updated. All chain files are collecting new data points, yet no update on the convergence test. What could be the problem?
Thanks in advance for the replies!

[Antony Lewis]

Sounds very long, normally chains should complete in order a day if set up well and standard speed likelihood. I would look at the output, I see you have Debug on so should have lots of tracking info. You can also plot the results in getdist to see if anything obviously going wrong.

[Ali Rida Khalife]

Thank you Antony! I didn't find any weird message in the log file. Could it be an MPI problem? I will plot the chains in see what I get.
Speaking of Debug, my log files are quite heavy (~50GB) when I set Debug: True, while they are normal (~0.1 MB) otherwise. This was not the case in older versions of COBAYA. Is there a way I can control the amount of output in Debug?

[Ali Rida Khalife]

Problem Solved
Turned out to be an MPI problem. If anyone ever face such a problem, make sure you load the appropriate intel openmpi module