From 0b00553b100250b02b12a0b93e44a55b1f70af14 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Thu, 25 Jun 2020 23:03:20 -0400 Subject: [PATCH 001/199] fixed gas_scatter_proportion tested --- mermithid/misc/ComplexLineShapeUtilities.py | 7 +- .../misc/MultiGasComplexLineShape.py | 550 ++++++++++++++++++ test_analysis/Complex_line_shape_fitter.py | 17 +- 3 files changed, 565 insertions(+), 9 deletions(-) create mode 100644 mermithid/processors/misc/MultiGasComplexLineShape.py diff --git a/mermithid/misc/ComplexLineShapeUtilities.py b/mermithid/misc/ComplexLineShapeUtilities.py index 074013d0..5fc40fa9 100644 --- a/mermithid/misc/ComplexLineShapeUtilities.py +++ b/mermithid/misc/ComplexLineShapeUtilities.py @@ -58,6 +58,10 @@ def aseev_func_tail(energy_loss_array, gas_type): A2, omeg2, eps2 = 0.195, 14.13, 10.60 elif gas_type=="Kr": A2, omeg2, eps2 = 0.4019, 22.31, 16.725 + elif gas_type=="He": + A2, omeg2, eps2 = 0.1187, 33.40, 10.43 + elif gas_type=="Ar": + A2, omeg2, eps2 = 0.3344, 21.91, 21.14 return A2*omeg2**2./(omeg2**2.+4*(energy_loss_array-eps2)**2.) #convert oscillator strength into energy loss spectrum @@ -65,7 +69,8 @@ def get_eloss_spec(e_loss, oscillator_strength, kr_17keV_line): #energies in eV kinetic_en = kr_17keV_line * 1000 e_rydberg = 13.605693009 #rydberg energy (eV) a0 = 5.291772e-11 #bohr radius - return np.where(e_loss>0 , 4.*np.pi*a0**2 * e_rydberg / (kinetic_en * e_loss) * oscillator_strength * np.log(4. * kinetic_en * e_loss / (e_rydberg**3.) ), 0) + argument_of_log = np.where(e_loss > 0, 4. * kinetic_en * e_loss / (e_rydberg**3.) , 1e-5) + return np.where(e_loss>0 , 4.*np.pi*a0**2 * e_rydberg / (kinetic_en * e_loss) * oscillator_strength * np.log(argument_of_log), 0) # Takes only the nonzero bins of a histogram def get_only_nonzero_bins(bins,hist): diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py new file mode 100644 index 00000000..fcd11f4b --- /dev/null +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -0,0 +1,550 @@ +''' +Fits data to complex lineshape model. +Author: E. Machado, Y.-H. Sun, E. Novitski +Date: 4/8/20 + +This processor takes in frequency data in binned histogram and fit the histogram with two gas scattering complex line shape model. + +Configurable parameters: + +There are two options available for fitting: fix_scatter_proportion = True and False. +gases: array for names of the two gases involved in the scattering process. +max_scatter: max number of scatterings for only single gas scatterings. +max_comprehansive_scatter: max number of scatterings for all cross scatterings. +scatter_proportion: when fix_scatter_proportion is set as true, gives the fixed scatter proportion. +num_points_in_std_array: number of points for std_array defining how finely the scatter calculations are. +RF_ROI_MIN: can be found from meta data. +B_field: can be put in hand or found by position of the peak of the frequency histogram. +shake_spectrum_parameters_json_path: path to json file storing shake spectrum parameters. +path_to_osc_strength_files: path to oscillator strength files. +''' + +from __future__ import absolute_import + +import numpy as np +from scipy.optimize import curve_fit +from scipy.special import comb +from scipy import integrate , signal, interpolate +from itertools import product +from math import factorial +import os +import time +import sys +from morpho.utilities import morphologging, reader +from morpho.processors import BaseProcessor +from mermithid.misc import Constants, ComplexLineShapeUtilities, ConversionFunctions + +logger = morphologging.getLogger(__name__) + + + +__all__ = [] +__all__.append(__name__) + +class MultiGasComplexLineShape(BaseProcessor): + + def InternalConfigure(self, params): + ''' + Configure + ''' + # Read other parameters + self.bins_choice = reader.read_param(params, 'bins_choice', []) + self.gases = reader.read_param(params, 'gases', ["H2", "Kr", "He", "Ar"]) + self.max_scatters = reader.read_param(params, 'max_scatters', 20) + self.fix_scatter_proportion = reader.read_param(params, 'fix_scatter_proportion', True) + if self.fix_scatter_proportion == True: + self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) + # This is an important parameter which determines how finely resolved + # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown + self.num_points_in_std_array = reader.read_param(params, 'num_points_in_std_array', 10000) + self.RF_ROI_MIN = reader.read_param(params, 'RF_ROI_MIN', 25850000000.0) + self.B_field = reader.read_param(params, 'B_field', 0.957810722501) + self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') + self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') + self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') + + if not os.path.exists(self.shake_spectrum_parameters_json_path): + raise IOError('Shake spectrum path does not exist') + if not os.path.exists(self.path_to_osc_strengths_files): + raise IOError('Path to osc strengths files does not exist') + + def InternalRun(self): + + # Read shake parameters from JSON file + self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) + + # number_of_events = len(self.data['StartFrequency']) + # self.results = number_of_events + + a = self.data['StartFrequency'] + + # fit with shake spectrum + data_hist_freq, freq_bins= np.histogram(a,bins=self.bins_choice) + histogram = data_hist_freq + bins = freq_bins + guess = np.where(np.array(histogram) == np.max(histogram))[0][0] + kr17kev_in_hz = guess*(bins[1]-bins[0])+bins[0] + #self.B_field = B(17.8, kr17kev_in_hz + 0) + if self.fix_scatter_proportion == True: + self.results = self.fit_data_1(freq_bins, data_hist_freq) + else: + self.results = self.fit_data(freq_bins, data_hist_freq) + + return True + + + # Establishes a standard energy loss array (SELA) from -1000 eV to 1000 eV + # with number of points equal to self.num_points_in_std_array. All convolutions + # will be carried out on this particular discretization + def std_eV_array(self): + emin = -1000 + emax = 1000 + array = np.linspace(emin,emax,self.num_points_in_std_array) + return array + + # A lorentzian line centered at 0 eV, with 2.83 eV width on the SELA + def std_lorenztian_17keV(self): + x_array = self.std_eV_array() + ans = lorentzian(x_array,0,kr_line_width) + return ans + + # A gaussian centered at 0 eV with variable width, on the SELA + def std_gaussian(self, sigma): + x_array = self.std_eV_array() + ans = ComplexLineShapeUtilities.gaussian(x_array,1,sigma,0) + return ans + # normalizes a function, but depends on binning. + # Only to be used for functions evaluated on the SELA + def normalize(self, f): + x_arr = self.std_eV_array() + f_norm = integrate.simps(f,x=x_arr) + f_normed = f/f_norm + return f_normed + + # Function for energy loss from a single scatter of electrons by + # V.N. Aseev et al. 2000 + # This function does the work of combining fit_func1 and fit_func2 by + # finding the point where they intersect. + # Evaluated on the SELA + def single_scatter_f(self, gas_type): + energy_loss_array = self.std_eV_array() + f = 0 * energy_loss_array + + input_filename = self.path_to_osc_strengths_files + gas_type + "OscillatorStrength.txt" + energy_fOsc = ComplexLineShapeUtilities.read_oscillator_str_file(input_filename) + fData = interpolate.interp1d(energy_fOsc[0], energy_fOsc[1], kind='linear') + for i in range(len(energy_loss_array)): + if energy_loss_array[i] < energy_fOsc[0][0]: + f[i] = 0 + elif energy_loss_array[i] <= energy_fOsc[0][-1]: + f[i] = fData(energy_loss_array[i]) + else: + f[i] = ComplexLineShapeUtilities.aseev_func_tail(energy_loss_array[i], gas_type) + + f_e_loss = ComplexLineShapeUtilities.get_eloss_spec(energy_loss_array, f, Constants.kr_k_line_e()) + f_normed = self.normalize(f_e_loss) + return f_normed + + # Convolves a function with the single scatter function, on the SELA + def another_scatter(self, input_spectrum, gas_type): + single = self.single_scatter_f(gas_type) + f = signal.convolve(single,input_spectrum,mode='same') + f_normed = self.normalize(f) + return f_normed + + # Convolves the scatter functions and saves + # the results to a .npy file. + def generate_scatter_convolution_file(self): + t = time.time() + scatter_spectra_single_gas = {} + for gas_type in self.gases: + scatter_spectra_single_gas[gas_type] = {} + first_scatter = self.single_scatter_f(gas_type) + scatter_num_array = range(2, self.max_scatters+1) + current_scatter = first_scatter + scatter_spectra_single_gas[gas_type][str(1).zfill(2)] = current_scatter + # x = std_eV_array() # diagnostic + for i in scatter_num_array: + current_scatter = self.another_scatter(current_scatter, gas_type) + scatter_spectra_single_gas[gas_type][str(i).zfill(2)] = current_scatter + N = len(self.gases) + scatter_spectra = {} + for M in range(1, self.max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + mark_first_nonzero_component = 0 + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + if component == 0: + continue + else: + if mark_first_nonzero_component == 0: + current_full_scatter = scatter_spectra_single_gas[gas_type][str(component).zfill(2)] + mark_first_nonzero_component = 1 + else: + scatter_to_add = scatter_spectra_single_gas[gas_type][str(component).zfill(2)] + current_full_scatter = self.normalize(signal.convolve(current_full_scatter, scatter_to_add, mode='same')) + scatter_spectra[entry_str] = current_full_scatter + logger.info(len(scatter_spectra)) + logger.info(self.path_to_scatter_spectra_file + 'scatter_spectra.npy') + np.save(self.path_to_scatter_spectra_file + 'scatter_spectra.npy', scatter_spectra) + elapsed = time.time() - t + logger.info('Files generated in '+str(elapsed)+'s') + return + + # Checks for the existence of a directory called 'scatter_spectra_file' + # and checks that this directory contains the scatter spectra files. + # If not, this function calls generate_scatter_convolution_file. + # This function also checks to make sure that the scatter file have the correct + # number of entries and correct number of points in the SELA, and if not, it generates a fresh file. + # When the variable regenerate is set as True, it generates a fresh file + def check_existence_of_scatter_file(self, regenerate = True): + gases = self.gases + if regenerate == True: + logger.info('generate fresh scatter file') + self.generate_scatter_convolution_file() + else: + directory = os.listdir(self.path_to_scatter_spectra_files) + strippeddirs = [s.strip('\n') for s in directory] + if 'scatter_spectra.npy' not in strippeddirs: + self.generate_scatter_convolution_file() + test_file = self.path_to_scatter_spectra_files+'scatter_spectra.npy' + test_dict = np.load(test_file, allow_pickle = True) + M = self.max_scatters + N = len(self.gases) + if len(test_dict.item()) != comb(M + N -1, N -1): + logger.info('Number of scatter combinations not matching, generating fresh files') + self.generate_scatter_convolution_file() + return + + # Given a function evaluated on the SELA, convolves it with a gaussian + def convolve_gaussian(self, func_to_convolve,gauss_FWHM_eV): + sigma = ComplexLineShapeUtilities.gaussian_FWHM_to_sigma(gauss_FWHM_eV) + resolution_f = self.std_gaussian(sigma) + ans = signal.convolve(resolution_f,func_to_convolve,mode='same') + ans_normed = self.normalize(ans) + return ans_normed + + def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitted_peak='shake'): + gases = self.gases + max_scatters = self.max_scatters + current_path = self.path_to_scatter_spectra_file + # check_existence_of_scatter_files() + #filenames = list_files('scatter_spectra_files') + p = np.zeros(len(gases)) + p[0:-1] = scatter_proportion + p[-1] = 1 - sum(scatter_proportion) + scatter_spectra = np.load( + current_path + 'scatter_spectra.npy', allow_pickle = True + ) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_gaussian(current_working_spectrum, gauss_FWHM_eV) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += current_working_spectrum + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + print(combination) + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = normalize(sp.signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(len(self.gases))): + coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M + current_full_spectrum += coefficient*current_working_spectrum + return current_full_spectrum + + # Produces a spectrum in real energy that can now be evaluated off of the SELA. + #def spectrum_func(x_keV,FWHM_G_eV,line_pos_keV,scatter_prob,amplitude): + def spectrum_func(self, x_keV, *p0): + x_eV = x_keV*1000. + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + en_array_rev = ComplexLineShapeUtilities.flip_array(-1*en_loss_array) + f = np.zeros(len(x_keV)) + f_intermediate = np.zeros(len(x_keV)) + + FWHM_G_eV = p0[0] + line_pos_keV = p0[1] + amplitude = p0[2] + prob_parameter = p0[3] + N = len(self.gases) + scatter_proportion = p0[4:3+N] + + line_pos_eV = line_pos_keV*1000. + x_eV_minus_line = x_eV - line_pos_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_keV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum(FWHM_G_eV, prob_parameter, scatter_proportion) + full_spectrum_rev = ComplexLineShapeUtilities.flip_array(full_spectrum) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum_rev) + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + return f + + # Call this function to fit a histogram of start frequencies with the model. + # Note that the data_hist_freq should be the StartFrequencies as given by katydid, + # which will be from ~0 MHZ to ~100 MHz. You must also pass this function the + # self.RF_ROI_MIN value from the metadata file of your data. + # You must also supply a guess for the self.B_field present for the run; + # 0.959 T is usually sufficient. + + def fit_data(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + bins_keV = ConversionFunctions.Energy(bins_Hz, self.B_field)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + data_hist = ComplexLineShapeUtilities.flip_array(data_hist_freq) + #data_hist_err = ComplexLineShapeUtilities.get_hist_err_bins(data_hist) + bins_keV_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_keV, data_hist) + # Bounds for curve_fit + FWHM_eV_min = 1e-5 + FWHM_eV_max = (bins_keV[len(bins_keV)-1] - bins_keV[0])*1000 + line_pos_keV_min = bins_keV[0] + line_pos_keV_max = bins_keV[len(bins_keV)-1] + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist)*3 + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + # Initial guesses for curve_fit + FWHM_guess = 5 + line_pos_guess = bins_keV[np.argmax(data_hist)] + amplitude_guess = np.sum(data_hist)/2 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + N = len(gases) + p0_guess = [FWHM_guess, line_pos_guess, amplitude_guess, prob_parameter_guess] + [scatter_proportion_guess]*(N-1) + p0_bounds = ([FWHM_eV_min, line_pos_keV_min, amplitude_min, prob_parameter_min] + [scatter_proportion_min]*(N-1), + [FWHM_eV_max, line_pos_keV_max, amplitude_max, prob_parameter_max] + [scatter_proportion_max]*(N-1)) + # Actually do the fitting + params , cov = curve_fit(self.spectrum_func, bins_keV_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) + # Name each of the resulting parameters and errors + ################### Generalize to N Gases ########################### + FWHM_G_eV_fit = params[0] + line_pos_keV_fit = params[1] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[2] + prob_parameter_fit = params[3] + scatter_proportion_fit = params[4:3+N]+[1- sum(params[4:3+N])] + total_counts_fit = amplitude_fit + + perr = np.sqrt(np.diag(cov)) + FWHM_eV_G_fit_err = perr[0] + line_pos_keV_fit_err = perr[1] + amplitude_fit_err = perr[2] + prob_parameter_fit_err = perr[3] + scatter_proportion_fit_err = perr[4:3+N]+[np.sqrt(perr[4:3+N]**2)] + total_counts_fit_err = amplitude_fit_err + + fit = self.spectrum_func(bins_keV,*params) + + line_pos_Hz_fit , line_pos_Hz_fit_err = ComplexLineShapeUtilities.energy_guess_to_frequency(line_pos_keV_fit, line_pos_keV_fit_err, self.B_field) + B_field_fit , B_field_fit_err = ComplexLineShapeUtilities.central_frequency_to_B_field(line_pos_Hz_fit, line_pos_Hz_fit_err) + fit_Hz = ComplexLineShapeUtilities.flip_array(fit) + bins_keV = bins_keV - line_pos_keV_fit + Constants.kr_k_line_e()/1000 + FWHM_eV_fit = FWHM_G_eV_fit + FWHM_eV_fit_err = FWHM_eV_G_fit_err + elapsed = time.time() - t + output_string = '\n' + output_string += 'B field = {:.6e}'.format(B_field_fit)+' +/- '+ '{:.2e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Gaussian FWHM = '+str(round(FWHM_G_eV_fit,2))+' +/- '+str(round(FWHM_eV_G_fit_err,2))+' eV\n' + output_string += '-----------------\n' + output_string += 'Line position \n= '+str(round(line_pos_Hz_fit,2))+' +/- '+str(round(line_pos_Hz_fit_err,2))+' Hz\n' + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)\ + +' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' + output_string += '-----------------\n' + for i in range(len(gases)): + output_string += '{} Scatter proportion \n= '.format(gases[i]) + "{:.2e}".format(scatter_proportion_fit[i])\ + +' +/- ' + "{:.2e}".format(scatter_proportion_fit_err[i])+'\n' + output_string += '-----------------\n' + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'cov': cov, + 'bins_keV': bins_keV, + 'fit': fit, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'FWHM_eV_fit': FWHM_eV_fit, + 'FWHM_eV_fit_err': FWHM_eV_fit_err, + 'line_pos_Hz_fit': line_pos_Hz_fit, + 'line_pos_Hz_fit_err': line_pos_Hz_fit_err, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'prob_parameter_fit': prob_parameter_fit, + 'prob_parameter_fit_err': prob_parameter_fit_err, + 'scatter_proportion_fit': scatter_proportion_fit, + 'scatter_proportion_fit_err': scatter_proportion_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq + } + return dictionary_of_fit_results + + def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak='shake'): + gases = self.gases + current_path = self.path_to_scatter_spectra_file + # check_existence_of_scatter_files() + #filenames = list_files('scatter_spectra_files') + p = np.zeros(len(gases)) + p = self.scatter_proportion + scatter_spectra = np.load( + current_path + 'scatter_spectra.npy', allow_pickle = True + ) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_gaussian(current_working_spectrum, gauss_FWHM_eV) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += current_working_spectrum + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(len(self.gases))): + coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M + current_full_spectrum += coefficient*current_working_spectrum + return current_full_spectrum + + def spectrum_func_1(self, x_keV, *p0): + x_eV = x_keV*1000. + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + en_array_rev = ComplexLineShapeUtilities.flip_array(-1*en_loss_array) + f = np.zeros(len(x_keV)) + f_intermediate = np.zeros(len(x_keV)) + + FWHM_G_eV = p0[0] + line_pos_keV = p0[1] + amplitude = p0[2] + prob_parameter = p0[3] + + line_pos_eV = line_pos_keV*1000. + x_eV_minus_line = x_eV - line_pos_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_keV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_1(FWHM_G_eV, prob_parameter,) + full_spectrum_rev = ComplexLineShapeUtilities.flip_array(full_spectrum) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum_rev) + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + return f + + def fit_data_1(self, freq_bins, data_hist_freq): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + bins_keV = ConversionFunctions.Energy(bins_Hz, self.B_field)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + data_hist = ComplexLineShapeUtilities.flip_array(data_hist_freq) + #data_hist_err = ComplexLineShapeUtilities.get_hist_err_bins(data_hist) + bins_keV_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_keV, data_hist) + # Bounds for curve_fit + FWHM_eV_min = 1e-5 + FWHM_eV_max = (bins_keV[len(bins_keV)-1] - bins_keV[0])*1000 + line_pos_keV_min = bins_keV[0] + line_pos_keV_max = bins_keV[len(bins_keV)-1] + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist)*3 + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + # Initial guesses for curve_fit + FWHM_guess = 5 + line_pos_guess = bins_keV[np.argmax(data_hist)] + amplitude_guess = np.sum(data_hist)/2 + prob_parameter_guess = 0.5 + p0_guess = [FWHM_guess, line_pos_guess, amplitude_guess, prob_parameter_guess] + p0_bounds = ([FWHM_eV_min, line_pos_keV_min, amplitude_min, prob_parameter_min], + [FWHM_eV_max, line_pos_keV_max, amplitude_max, prob_parameter_max]) + # Actually do the fitting + params , cov = curve_fit(self.spectrum_func_1, bins_keV_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) + # Name each of the resulting parameters and errors + ################### Generalize to N Gases ########################### + FWHM_G_eV_fit = params[0] + line_pos_keV_fit = params[1] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[2] + prob_parameter_fit = params[3] + total_counts_fit = amplitude_fit + + perr = np.sqrt(np.diag(cov)) + FWHM_eV_G_fit_err = perr[0] + line_pos_keV_fit_err = perr[1] + amplitude_fit_err = perr[2] + prob_parameter_fit_err = perr[3] + total_counts_fit_err = amplitude_fit_err + + fit = self.spectrum_func_1(bins_keV,*params) + + line_pos_Hz_fit , line_pos_Hz_fit_err = ComplexLineShapeUtilities.energy_guess_to_frequency(line_pos_keV_fit, line_pos_keV_fit_err, self.B_field) + B_field_fit , B_field_fit_err = ComplexLineShapeUtilities.central_frequency_to_B_field(line_pos_Hz_fit, line_pos_Hz_fit_err) + fit_Hz = ComplexLineShapeUtilities.flip_array(fit) + bins_keV = bins_keV - line_pos_keV_fit + Constants.kr_k_line_e()/1000 + FWHM_eV_fit = FWHM_G_eV_fit + FWHM_eV_fit_err = FWHM_eV_G_fit_err + elapsed = time.time() - t + output_string = '\n' + output_string += 'B field = {:.6e}'.format(B_field_fit)+' +/- '+ '{:.2e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Gaussian FWHM = '+str(round(FWHM_G_eV_fit,2))+' +/- '+str(round(FWHM_eV_G_fit_err,2))+' eV\n' + output_string += '-----------------\n' + output_string += 'Line position \n= '+str(round(line_pos_Hz_fit,2))+' +/- '+str(round(line_pos_Hz_fit_err,2))+' Hz\n' + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)\ + +' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' + output_string += '-----------------\n' + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'cov': cov, + 'bins_keV': bins_keV, + 'fit': fit, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'FWHM_eV_fit': FWHM_eV_fit, + 'FWHM_eV_fit_err': FWHM_eV_fit_err, + 'line_pos_Hz_fit': line_pos_Hz_fit, + 'line_pos_Hz_fit_err': line_pos_Hz_fit_err, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'prob_parameter_fit': prob_parameter_fit, + 'prob_parameter_fit_err': prob_parameter_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq + } + return dictionary_of_fit_results \ No newline at end of file diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 352288ef..75c20ace 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -15,12 +15,12 @@ class ComplexLineShapeTests(unittest.TestCase): def test_complex_lineshape(self): from mermithid.processors.IO import IOCicadaProcessor - from mermithid.processors.misc.KrComplexLineShape import KrComplexLineShape + from mermithid.processors.misc.MultiGasComplexLineShape import MultiGasComplexLineShape reader_config = { "action": "read", - "filename": "/host/ShallowTrap8603-8669.root", + "filename": "/host/march_2020_kr_calibration_channel_b_merged.root", "object_type": "TMultiTrackEventData", "object_name": "multiTrackEvents:Event", "use_katydid": False, @@ -28,11 +28,11 @@ def test_complex_lineshape(self): } complexLineShape_config = { 'bins_choice': np.linspace(0,90e6,1000), - 'gases': ["H2","Kr"], - 'max_scatters': 20, + 'gases': ["H2","Kr","He","Ar"], + 'max_scatters': 18, 'fix_scatter_proportion': True, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below - 'gas1_scatter_proportion': 0.8, + 'gas_scatter_proportion': [0.61, 0.04, 0.34, 0.01], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -40,11 +40,12 @@ def test_complex_lineshape(self): 'B_field': 0.957810722501, # shake_spectrum_parameters.json and oscillator strength data can be found at https://github.com/project8/scripts/tree/master/yuhao/line_shape_fitting/data 'shake_spectrum_parameters_json_path': '../mermithid/misc/shake_spectrum_parameters.json', - 'path_to_osc_strengths_files': '/host/' + 'path_to_osc_strengths_files': '/host/', + 'path_to_scatter_spectra_file': '/host/' } b = IOCicadaProcessor("reader") - complexLineShape = KrComplexLineShape("complexLineShape") + complexLineShape = MultiGasComplexLineShape("complexLineShape") b.Configure(reader_config) complexLineShape.Configure(complexLineShape_config) @@ -73,7 +74,7 @@ def test_complex_lineshape(self): plt.legend(loc = 'upper left', fontsize = 12) plt.xlabel('frequency GHz') plt.title('fit with shake spectrum 2 gas scattering') - plt.savefig('fit_shake_2_gas_0.png') + plt.savefig('/host/plots/fit_shake_2_gas_0.png') if __name__ == '__main__': From 9bf9ca2d49927efd50d718f7139846b60bd220ec Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 8 Jul 2020 11:43:39 -0400 Subject: [PATCH 002/199] add chi2 calculation --- .../misc/MultiGasComplexLineShape.py | 47 ++++++++++++++----- test_analysis/Complex_line_shape_fitter.py | 14 +++++- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index fcd11f4b..91982e4f 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -186,9 +186,7 @@ def generate_scatter_convolution_file(self): else: scatter_to_add = scatter_spectra_single_gas[gas_type][str(component).zfill(2)] current_full_scatter = self.normalize(signal.convolve(current_full_scatter, scatter_to_add, mode='same')) - scatter_spectra[entry_str] = current_full_scatter - logger.info(len(scatter_spectra)) - logger.info(self.path_to_scatter_spectra_file + 'scatter_spectra.npy') + scatter_spectra[entry_str] = current_full_scatter np.save(self.path_to_scatter_spectra_file + 'scatter_spectra.npy', scatter_spectra) elapsed = time.time() - t logger.info('Files generated in '+str(elapsed)+'s') @@ -212,9 +210,8 @@ def check_existence_of_scatter_file(self, regenerate = True): self.generate_scatter_convolution_file() test_file = self.path_to_scatter_spectra_files+'scatter_spectra.npy' test_dict = np.load(test_file, allow_pickle = True) - M = self.max_scatters N = len(self.gases) - if len(test_dict.item()) != comb(M + N -1, N -1): + if len(test_dict.item()) != sum([comb(M + N -1, N -1) for M in range(1, max_scatters+1)]): logger.info('Number of scatter combinations not matching, generating fresh files') self.generate_scatter_convolution_file() return @@ -262,7 +259,7 @@ def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitt coefficient = factorial(sum(combination)) for component, i in zip(combination, range(len(self.gases))): coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M - current_full_spectrum += coefficient*current_working_spectrum + current_full_spectrum += q*coefficient*current_working_spectrum return current_full_spectrum # Produces a spectrum in real energy that can now be evaluated off of the SELA. @@ -331,7 +328,7 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): N = len(gases) p0_guess = [FWHM_guess, line_pos_guess, amplitude_guess, prob_parameter_guess] + [scatter_proportion_guess]*(N-1) p0_bounds = ([FWHM_eV_min, line_pos_keV_min, amplitude_min, prob_parameter_min] + [scatter_proportion_min]*(N-1), - [FWHM_eV_max, line_pos_keV_max, amplitude_max, prob_parameter_max] + [scatter_proportion_max]*(N-1)) + [FWHM_eV_max, line_pos_keV_max, amplitude_max, prob_parameter_max] + [scatter_proportion_max]*(N-1) ) # Actually do the fitting params , cov = curve_fit(self.spectrum_func, bins_keV_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) # Name each of the resulting parameters and errors @@ -360,9 +357,20 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): bins_keV = bins_keV - line_pos_keV_fit + Constants.kr_k_line_e()/1000 FWHM_eV_fit = FWHM_G_eV_fit FWHM_eV_fit_err = FWHM_eV_G_fit_err + + nonzero_bins_index = np.where(data_hist_freq != 0)[0] + zero_bins_index = np.where(data_hist_freq == 0)[0] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_zero = fit_Hz['fit_Hz'][zero_bins_index] + data_Hz_zero = data_hist_freq[zero_bins_index] + chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) + reduced_chi2 = chi2/(len(data_hist_freq)-4-len(self.gases)+1) elapsed = time.time() - t output_string = '\n' - output_string += 'B field = {:.6e}'.format(B_field_fit)+' +/- '+ '{:.2e} T\n'.format(B_field_fit_err) + output_string += 'Reduced chi^2 = {:.2e}\n$'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) output_string += '-----------------\n' output_string += 'Gaussian FWHM = '+str(round(FWHM_G_eV_fit,2))+' +/- '+str(round(FWHM_eV_G_fit_err,2))+' eV\n' output_string += '-----------------\n' @@ -373,8 +381,8 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)\ +' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' output_string += '-----------------\n' - for i in range(len(gases)): - output_string += '{} Scatter proportion \n= '.format(gases[i]) + "{:.2e}".format(scatter_proportion_fit[i])\ + for i in range(len(self.gases)): + output_string += '{} Scatter proportion \n= '.format(gases[i]) + "{:.8e}".format(scatter_proportion_fit[i])\ +' +/- ' + "{:.2e}".format(scatter_proportion_fit_err[i])+'\n' output_string += '-----------------\n' output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' @@ -397,7 +405,8 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): 'scatter_proportion_fit_err': scatter_proportion_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, - 'data_hist_freq': data_hist_freq + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results @@ -514,9 +523,20 @@ def fit_data_1(self, freq_bins, data_hist_freq): bins_keV = bins_keV - line_pos_keV_fit + Constants.kr_k_line_e()/1000 FWHM_eV_fit = FWHM_G_eV_fit FWHM_eV_fit_err = FWHM_eV_G_fit_err + + nonzero_bins_index = np.where(data_hist_freq != 0)[0] + zero_bins_index = np.where(data_hist_freq == 0)[0] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_zero = fit_Hz[zero_bins_index] + data_Hz_zero = data_hist_freq[zero_bins_index] + chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) + reduced_chi2 = chi2/(len(data_hist_freq)-4) elapsed = time.time() - t output_string = '\n' - output_string += 'B field = {:.6e}'.format(B_field_fit)+' +/- '+ '{:.2e} T\n'.format(B_field_fit_err) + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) output_string += '-----------------\n' output_string += 'Gaussian FWHM = '+str(round(FWHM_G_eV_fit,2))+' +/- '+str(round(FWHM_eV_G_fit_err,2))+' eV\n' output_string += '-----------------\n' @@ -545,6 +565,7 @@ def fit_data_1(self, freq_bins, data_hist_freq): 'prob_parameter_fit_err': prob_parameter_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, - 'data_hist_freq': data_hist_freq + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results \ No newline at end of file diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 75c20ace..dd8a521c 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -73,8 +73,18 @@ def test_complex_lineshape(self): plt.plot(results['bins_Hz']/1e9, results['fit_Hz'], label = results['output_string'], alpha = 0.7) plt.legend(loc = 'upper left', fontsize = 12) plt.xlabel('frequency GHz') - plt.title('fit with shake spectrum 2 gas scattering') - plt.savefig('/host/plots/fit_shake_2_gas_0.png') + plot_title = 'fit with {} gas scattering'.format(len(complexLineShape_config['gases'])) + if complexLineShape_config['fix_scatter_proportion'] == True: + str_gas_scatter_proportion = '' + for i in range(len(complexLineShape_config['gases'])): + str_gas_scatter_proportion += complexLineShape_config['gases'][i] + str_gas_scatter_proportion += ': ' + str_gas_scatter_proportion += str(complexLineShape_config['gas_scatter_proportion'][i]) + str_gas_scatter_proportion += ' ' + plot_title += '\n with fixed scatter proportion \n {}'.format(str_gas_scatter_proportion) + plt.title(plot_title) + plt.tight_layout() + plt.savefig('/host/plots/fit_with_{}_gas_scattering.png'.format(len(complexLineShape_config['gases']))) if __name__ == '__main__': From bb0bc61cee5ce424c0147bcde553c04d943e3486 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Sat, 11 Jul 2020 14:25:15 -0400 Subject: [PATCH 003/199] fix the checking of scatter spectra file --- .../processors/misc/MultiGasComplexLineShape.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 91982e4f..5ea2f1e5 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -187,7 +187,7 @@ def generate_scatter_convolution_file(self): scatter_to_add = scatter_spectra_single_gas[gas_type][str(component).zfill(2)] current_full_scatter = self.normalize(signal.convolve(current_full_scatter, scatter_to_add, mode='same')) scatter_spectra[entry_str] = current_full_scatter - np.save(self.path_to_scatter_spectra_file + 'scatter_spectra.npy', scatter_spectra) + np.save(os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy'), scatter_spectra) elapsed = time.time() - t logger.info('Files generated in '+str(elapsed)+'s') return @@ -208,12 +208,19 @@ def check_existence_of_scatter_file(self, regenerate = True): strippeddirs = [s.strip('\n') for s in directory] if 'scatter_spectra.npy' not in strippeddirs: self.generate_scatter_convolution_file() - test_file = self.path_to_scatter_spectra_files+'scatter_spectra.npy' + test_file = os.path.join(self.path_to_scatter_spectra_files, 'scatter_spectra.npy') test_dict = np.load(test_file, allow_pickle = True) N = len(self.gases) if len(test_dict.item()) != sum([comb(M + N -1, N -1) for M in range(1, max_scatters+1)]): logger.info('Number of scatter combinations not matching, generating fresh files') self.generate_scatter_convolution_file() + test_dict = np.load(test_file, allow_pickle = True) + gas_str = gases[0] + '01' + for gas in self.gases[1:]: + gas_str += gas + '00' + if gas_str not in list(test_dict.item().keys()): + print('Gas species not matching, generating fresh files') + generate_scatter_convolution_files() return # Given a function evaluated on the SELA, convolves it with a gaussian @@ -233,8 +240,9 @@ def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitt p = np.zeros(len(gases)) p[0:-1] = scatter_proportion p[-1] = 1 - sum(scatter_proportion) + scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') scatter_spectra = np.load( - current_path + 'scatter_spectra.npy', allow_pickle = True + os.path.join(scatter_spectra_file_path, allow_pickle = True ) en_array = self.std_eV_array() current_full_spectrum = np.zeros(len(en_array)) @@ -417,8 +425,9 @@ def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak='shake'): #filenames = list_files('scatter_spectra_files') p = np.zeros(len(gases)) p = self.scatter_proportion + scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') scatter_spectra = np.load( - current_path + 'scatter_spectra.npy', allow_pickle = True + scatter_spectra_file_path, allow_pickle = True ) en_array = self.std_eV_array() current_full_spectrum = np.zeros(len(en_array)) From fab92a7c47d326c72350b05cd2487727ca8185ec Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Sat, 11 Jul 2020 14:37:10 -0400 Subject: [PATCH 004/199] fix the checking of scatter spectra file --- mermithid/processors/misc/MultiGasComplexLineShape.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 5ea2f1e5..8667218b 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -198,20 +198,20 @@ def generate_scatter_convolution_file(self): # This function also checks to make sure that the scatter file have the correct # number of entries and correct number of points in the SELA, and if not, it generates a fresh file. # When the variable regenerate is set as True, it generates a fresh file - def check_existence_of_scatter_file(self, regenerate = True): + def check_existence_of_scatter_file(self, regenerate = False): gases = self.gases if regenerate == True: logger.info('generate fresh scatter file') self.generate_scatter_convolution_file() else: - directory = os.listdir(self.path_to_scatter_spectra_files) + directory = os.listdir(self.path_to_scatter_spectra_file) strippeddirs = [s.strip('\n') for s in directory] if 'scatter_spectra.npy' not in strippeddirs: self.generate_scatter_convolution_file() - test_file = os.path.join(self.path_to_scatter_spectra_files, 'scatter_spectra.npy') + test_file = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') test_dict = np.load(test_file, allow_pickle = True) N = len(self.gases) - if len(test_dict.item()) != sum([comb(M + N -1, N -1) for M in range(1, max_scatters+1)]): + if len(test_dict.item()) != sum([comb(M + N -1, N -1) for M in range(1, self.max_scatters+1)]): logger.info('Number of scatter combinations not matching, generating fresh files') self.generate_scatter_convolution_file() test_dict = np.load(test_file, allow_pickle = True) @@ -242,7 +242,7 @@ def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitt p[-1] = 1 - sum(scatter_proportion) scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') scatter_spectra = np.load( - os.path.join(scatter_spectra_file_path, allow_pickle = True + scatter_spectra_file_path, allow_pickle = True ) en_array = self.std_eV_array() current_full_spectrum = np.zeros(len(en_array)) From c173168b63f8ff1e7f469aeeee2a8781fd444d2d Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Sat, 11 Jul 2020 15:10:08 -0400 Subject: [PATCH 005/199] internal config error --- mermithid/processors/misc/MultiGasComplexLineShape.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 8667218b..4e25f98e 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -67,6 +67,7 @@ def InternalConfigure(self, params): raise IOError('Shake spectrum path does not exist') if not os.path.exists(self.path_to_osc_strengths_files): raise IOError('Path to osc strengths files does not exist') + return True def InternalRun(self): From fa62da5803bd08e339c8678ea23ac1cda26c5414 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Thu, 20 Aug 2020 16:48:11 -0400 Subject: [PATCH 006/199] fix magnetic field mismatch between notebook and mermithid result --- mermithid/misc/ComplexLineShapeUtilities.py | 2 +- test_analysis/Complex_line_shape_fitter.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/mermithid/misc/ComplexLineShapeUtilities.py b/mermithid/misc/ComplexLineShapeUtilities.py index 5fc40fa9..e85140c3 100644 --- a/mermithid/misc/ComplexLineShapeUtilities.py +++ b/mermithid/misc/ComplexLineShapeUtilities.py @@ -66,7 +66,7 @@ def aseev_func_tail(energy_loss_array, gas_type): #convert oscillator strength into energy loss spectrum def get_eloss_spec(e_loss, oscillator_strength, kr_17keV_line): #energies in eV - kinetic_en = kr_17keV_line * 1000 + kinetic_en = kr_17keV_line e_rydberg = 13.605693009 #rydberg energy (eV) a0 = 5.291772e-11 #bohr radius argument_of_log = np.where(e_loss > 0, 4. * kinetic_en * e_loss / (e_rydberg**3.) , 1e-5) diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index dd8a521c..e4b3e904 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -7,6 +7,8 @@ import numpy as np import unittest import matplotlib.pyplot as plt +from root_numpy import tree2array +import ROOT as r from morpho.utilities import morphologging, parser logger = morphologging.getLogger(__name__) @@ -27,12 +29,12 @@ def test_complex_lineshape(self): "variables": ['StartTimeInAcq','StartFrequency'] } complexLineShape_config = { - 'bins_choice': np.linspace(0,90e6,1000), - 'gases': ["H2","Kr","He","Ar"], - 'max_scatters': 18, + 'bins_choice': np.linspace(0e6, 90e6, 1000), + 'gases': ["H2","Kr"], + 'max_scatters': 20, 'fix_scatter_proportion': True, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below - 'gas_scatter_proportion': [0.61, 0.04, 0.34, 0.01], + 'gas_scatter_proportion': [0.61, 0.39], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -73,7 +75,7 @@ def test_complex_lineshape(self): plt.plot(results['bins_Hz']/1e9, results['fit_Hz'], label = results['output_string'], alpha = 0.7) plt.legend(loc = 'upper left', fontsize = 12) plt.xlabel('frequency GHz') - plot_title = 'fit with {} gas scattering'.format(len(complexLineShape_config['gases'])) + plot_title = 'fit shallow trap 7418 with {} gas scattering'.format(len(complexLineShape_config['gases'])) if complexLineShape_config['fix_scatter_proportion'] == True: str_gas_scatter_proportion = '' for i in range(len(complexLineShape_config['gases'])): @@ -84,7 +86,7 @@ def test_complex_lineshape(self): plot_title += '\n with fixed scatter proportion \n {}'.format(str_gas_scatter_proportion) plt.title(plot_title) plt.tight_layout() - plt.savefig('/host/plots/fit_with_{}_gas_scattering.png'.format(len(complexLineShape_config['gases']))) + plt.savefig('/host/plots/fit_shallow_trap_7418_with_{}_gas_scattering.png'.format(len(complexLineShape_config['gases']))) if __name__ == '__main__': From 96f1259466ab455378b602b4e372152089ca6f36 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Thu, 3 Sep 2020 08:43:48 -0400 Subject: [PATCH 007/199] add radiation loss --- mermithid/misc/ComplexLineShapeUtilities.py | 6 +-- .../misc/MultiGasComplexLineShape.py | 39 +++++++++++++------ 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/mermithid/misc/ComplexLineShapeUtilities.py b/mermithid/misc/ComplexLineShapeUtilities.py index e85140c3..5e186dfd 100644 --- a/mermithid/misc/ComplexLineShapeUtilities.py +++ b/mermithid/misc/ComplexLineShapeUtilities.py @@ -66,11 +66,11 @@ def aseev_func_tail(energy_loss_array, gas_type): #convert oscillator strength into energy loss spectrum def get_eloss_spec(e_loss, oscillator_strength, kr_17keV_line): #energies in eV - kinetic_en = kr_17keV_line + kinetic_en = kr_17keV_line e_rydberg = 13.605693009 #rydberg energy (eV) a0 = 5.291772e-11 #bohr radius - argument_of_log = np.where(e_loss > 0, 4. * kinetic_en * e_loss / (e_rydberg**3.) , 1e-5) - return np.where(e_loss>0 , 4.*np.pi*a0**2 * e_rydberg / (kinetic_en * e_loss) * oscillator_strength * np.log(argument_of_log), 0) + argument_of_log = np.where(e_loss > 0, 4. * kinetic_en * e_rydberg / (e_loss**2.) , 1e-5) + return np.where(e_loss>0 , 1./(e_loss) * oscillator_strength*np.log(argument_of_log), 0) # Takes only the nonzero bins of a histogram def get_only_nonzero_bins(bins,hist): diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 4e25f98e..ca7fc9e0 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -62,6 +62,7 @@ def InternalConfigure(self, params): self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') + self.path_to_missing_track_radiation_loss_data_numpy_file = '/host' if not os.path.exists(self.shake_spectrum_parameters_json_path): raise IOError('Shake spectrum path does not exist') @@ -152,6 +153,20 @@ def another_scatter(self, input_spectrum, gas_type): f = signal.convolve(single,input_spectrum,mode='same') f_normed = self.normalize(f) return f_normed + + def radiation_loss_f(self): + radiation_loss_data_file_path = self.path_to_missing_track_radiation_loss_data_numpy_file + '/missing_track_radiation_loss.npy' + data_for_missing_track_radiation_loss = np.load(radiation_loss_data_file_path, allow_pickle = True) + x_data_for_histogram = f_radiation_energy_loss_interp = data_for_missing_track_radiation_loss.item()['histogram_eV']['x_data'] + energy_loss_array = self.std_eV_array() + f_radiation_energy_loss = 0 * energy_loss_array + f_radiation_energy_loss_interp = data_for_missing_track_radiation_loss.item()['histogram_eV']['interp'] + for i in range(len(energy_loss_array)): + if energy_loss_array[i] >= x_data_for_histogram[0] and energy_loss_array[i] <= x_data_for_histogram[-1]: + f_radiation_energy_loss[i] = f_radiation_energy_loss_interp(energy_loss_array[i]) + else: + f_radiation_energy_loss[i] = 0 + return f_radiation_energy_loss # Convolves the scatter functions and saves # the results to a .npy file. @@ -159,14 +174,17 @@ def generate_scatter_convolution_file(self): t = time.time() scatter_spectra_single_gas = {} for gas_type in self.gases: + f_radiation_loss = self.radiation_loss_f() scatter_spectra_single_gas[gas_type] = {} first_scatter = self.single_scatter_f(gas_type) + first_scatter = self.normalize(signal.convolve(first_scatter, f_radiation_loss, mode = 'same')) scatter_num_array = range(2, self.max_scatters+1) current_scatter = first_scatter scatter_spectra_single_gas[gas_type][str(1).zfill(2)] = current_scatter # x = std_eV_array() # diagnostic for i in scatter_num_array: current_scatter = self.another_scatter(current_scatter, gas_type) + current_scatter = self.normalize(signal.convolve(current_scatter, f_radiation_loss, mode = 'same')) scatter_spectra_single_gas[gas_type][str(i).zfill(2)] = current_scatter N = len(self.gases) scatter_spectra = {} @@ -199,7 +217,7 @@ def generate_scatter_convolution_file(self): # This function also checks to make sure that the scatter file have the correct # number of entries and correct number of points in the SELA, and if not, it generates a fresh file. # When the variable regenerate is set as True, it generates a fresh file - def check_existence_of_scatter_file(self, regenerate = False): + def check_existence_of_scatter_file(self, regenerate = True): gases = self.gases if regenerate == True: logger.info('generate fresh scatter file') @@ -258,17 +276,16 @@ def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitt for M in range(1, self.max_scatters + 1): gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: - print(combination) entry_str = '' for component, gas_type in zip(combination, self.gases): entry_str += gas_type entry_str += str(component).zfill(2) current_working_spectrum = scatter_spectra.item()[entry_str] - current_working_spectrum = normalize(sp.signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) coefficient = factorial(sum(combination)) for component, i in zip(combination, range(len(self.gases))): coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M - current_full_spectrum += q*coefficient*current_working_spectrum + current_full_spectrum += coefficient*current_working_spectrum return current_full_spectrum # Produces a spectrum in real energy that can now be evaluated off of the SELA. @@ -334,7 +351,7 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): amplitude_guess = np.sum(data_hist)/2 prob_parameter_guess = 0.5 scatter_proportion_guess = 0.5 - N = len(gases) + N = len(self.gases) p0_guess = [FWHM_guess, line_pos_guess, amplitude_guess, prob_parameter_guess] + [scatter_proportion_guess]*(N-1) p0_bounds = ([FWHM_eV_min, line_pos_keV_min, amplitude_min, prob_parameter_min] + [scatter_proportion_min]*(N-1), [FWHM_eV_max, line_pos_keV_max, amplitude_max, prob_parameter_max] + [scatter_proportion_max]*(N-1) ) @@ -344,10 +361,10 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): ################### Generalize to N Gases ########################### FWHM_G_eV_fit = params[0] line_pos_keV_fit = params[1] - #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) amplitude_fit = params[2] prob_parameter_fit = params[3] - scatter_proportion_fit = params[4:3+N]+[1- sum(params[4:3+N])] + #starting at index 4, grabs every other entry. (which is how scattering probs are filled in for N gases) + scatter_proportion_fit = list(params[4:3+N])+[1- sum(params[4:3+N])] total_counts_fit = amplitude_fit perr = np.sqrt(np.diag(cov)) @@ -355,7 +372,7 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): line_pos_keV_fit_err = perr[1] amplitude_fit_err = perr[2] prob_parameter_fit_err = perr[3] - scatter_proportion_fit_err = perr[4:3+N]+[np.sqrt(perr[4:3+N]**2)] + scatter_proportion_fit_err = list(perr[4:3+N])+[np.sqrt(sum(perr[4:3+N]**2))] total_counts_fit_err = amplitude_fit_err fit = self.spectrum_func(bins_keV,*params) @@ -371,13 +388,13 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): zero_bins_index = np.where(data_hist_freq == 0)[0] fit_Hz_nonzero = fit_Hz[nonzero_bins_index] data_Hz_nonzero = data_hist_freq[nonzero_bins_index] - fit_Hz_zero = fit_Hz['fit_Hz'][zero_bins_index] + fit_Hz_zero = fit_Hz[zero_bins_index] data_Hz_zero = data_hist_freq[zero_bins_index] chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) reduced_chi2 = chi2/(len(data_hist_freq)-4-len(self.gases)+1) elapsed = time.time() - t output_string = '\n' - output_string += 'Reduced chi^2 = {:.2e}\n$'.format(reduced_chi2) + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) output_string += '-----------------\n' output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) output_string += '-----------------\n' @@ -391,7 +408,7 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): +' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' output_string += '-----------------\n' for i in range(len(self.gases)): - output_string += '{} Scatter proportion \n= '.format(gases[i]) + "{:.8e}".format(scatter_proportion_fit[i])\ + output_string += '{} Scatter proportion \n= '.format(self.gases[i]) + "{:.8e}".format(scatter_proportion_fit[i])\ +' +/- ' + "{:.2e}".format(scatter_proportion_fit_err[i])+'\n' output_string += '-----------------\n' output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' From a27dc40b15dcc3f7aa4235754882db478d7112ff Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 9 Sep 2020 19:27:57 -0400 Subject: [PATCH 008/199] check in before adding file for composite trap fitting --- mermithid/misc/ComplexLineShapeUtilities.py | 5 +- .../misc/MultiGasComplexLineShape.py | 187 ++++++++---------- test_analysis/Complex_line_shape_fitter.py | 10 +- 3 files changed, 87 insertions(+), 115 deletions(-) diff --git a/mermithid/misc/ComplexLineShapeUtilities.py b/mermithid/misc/ComplexLineShapeUtilities.py index 5e186dfd..2a0be53d 100644 --- a/mermithid/misc/ComplexLineShapeUtilities.py +++ b/mermithid/misc/ComplexLineShapeUtilities.py @@ -110,11 +110,10 @@ def energy_guess_to_frequency(energy_guess, energy_guess_err, B_field_guess): return frequency , frequency_err # Given a frequency and error, converts those to B field values assuming the line is the 17.8 keV line -def central_frequency_to_B_field(central_freq,central_freq_err): +def central_frequency_to_B_field(central_freq): const = (2.*np.pi*m_e)*(1+kr_17keV_line/mass_energy_electron)/e_charge B_field = const*central_freq - B_field_err = const*central_freq_err - return B_field , B_field_err + return B_field # given a FWHM for the lorentian component and the FWHM for the gaussian component, # this function estimates the FWHM of the resulting voigt distribution diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index ca7fc9e0..42d4934e 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -290,30 +290,28 @@ def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitt # Produces a spectrum in real energy that can now be evaluated off of the SELA. #def spectrum_func(x_keV,FWHM_G_eV,line_pos_keV,scatter_prob,amplitude): - def spectrum_func(self, x_keV, *p0): - x_eV = x_keV*1000. + def spectrum_func(self, bins_Hz, *p0): + B_field = p0[0] + FWHM_G_eV = p0[1] + amplitude = p0[2] + prob_parameter = p0[3] + N = len(self.gases) + scatter_proportion = p0[4:3+N] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) en_loss_array = self.std_eV_array() en_loss_array_min = en_loss_array[0] en_loss_array_max = en_loss_array[len(en_loss_array)-1] en_array_rev = ComplexLineShapeUtilities.flip_array(-1*en_loss_array) - f = np.zeros(len(x_keV)) - f_intermediate = np.zeros(len(x_keV)) + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) - FWHM_G_eV = p0[0] - line_pos_keV = p0[1] - amplitude = p0[2] - prob_parameter = p0[3] - N = len(self.gases) - scatter_proportion = p0[4:3+N] - - line_pos_eV = line_pos_keV*1000. - x_eV_minus_line = x_eV - line_pos_eV + x_eV_minus_line = Constants.kr_k_line_e() - x_eV zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] - nonzero_idx = [i for i in range(len(x_keV)) if i not in zero_idx] - + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + full_spectrum = self.make_spectrum(FWHM_G_eV, prob_parameter, scatter_proportion) - full_spectrum_rev = ComplexLineShapeUtilities.flip_array(full_spectrum) - f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum_rev) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum) f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) return f @@ -329,38 +327,34 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - bins_keV = ConversionFunctions.Energy(bins_Hz, self.B_field)/1000 - bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) - data_hist = ComplexLineShapeUtilities.flip_array(data_hist_freq) - #data_hist_err = ComplexLineShapeUtilities.get_hist_err_bins(data_hist) - bins_keV_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_keV, data_hist) + bins_Hz_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_Hz, data_hist_freq) + # Initial guesses for curve_fit + FWHM_guess = 5 + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) FWHM_eV_min = 1e-5 - FWHM_eV_max = (bins_keV[len(bins_keV)-1] - bins_keV[0])*1000 - line_pos_keV_min = bins_keV[0] - line_pos_keV_max = bins_keV[len(bins_keV)-1] + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) - ConversionFunctions.Energy(bins_Hz[-1], B_field_guess) amplitude_min = 1e-5 - amplitude_max = np.sum(data_hist)*3 + amplitude_max = np.sum(data_hist_freq)*3 prob_parameter_min = 1e-5 prob_parameter_max = 1 scatter_proportion_min = 1e-5 scatter_proportion_max = 1 - # Initial guesses for curve_fit - FWHM_guess = 5 - line_pos_guess = bins_keV[np.argmax(data_hist)] - amplitude_guess = np.sum(data_hist)/2 - prob_parameter_guess = 0.5 - scatter_proportion_guess = 0.5 N = len(self.gases) - p0_guess = [FWHM_guess, line_pos_guess, amplitude_guess, prob_parameter_guess] + [scatter_proportion_guess]*(N-1) - p0_bounds = ([FWHM_eV_min, line_pos_keV_min, amplitude_min, prob_parameter_min] + [scatter_proportion_min]*(N-1), - [FWHM_eV_max, line_pos_keV_max, amplitude_max, prob_parameter_max] + [scatter_proportion_max]*(N-1) ) + p0_guess = [B_field_guess, FWHM_guess, amplitude_guess, prob_parameter_guess] + [scatter_proportion_guess]*(N-1) + p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min] + [scatter_proportion_min]*(N-1), + [B_field_max, FWHM_eV_max, amplitude_max, prob_parameter_max] + [scatter_proportion_max]*(N-1) ) # Actually do the fitting - params , cov = curve_fit(self.spectrum_func, bins_keV_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) + params , cov = curve_fit(self.spectrum_func, bins_Hz_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) # Name each of the resulting parameters and errors ################### Generalize to N Gases ########################### - FWHM_G_eV_fit = params[0] - line_pos_keV_fit = params[1] + B_field_fit = params[0] + FWHM_eV_fit = params[1] amplitude_fit = params[2] prob_parameter_fit = params[3] #starting at index 4, grabs every other entry. (which is how scattering probs are filled in for N gases) @@ -368,21 +362,17 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): total_counts_fit = amplitude_fit perr = np.sqrt(np.diag(cov)) - FWHM_eV_G_fit_err = perr[0] - line_pos_keV_fit_err = perr[1] + B_field_fit_err = perr[0] + FWHM_eV_fit_err = perr[1] amplitude_fit_err = perr[2] prob_parameter_fit_err = perr[3] scatter_proportion_fit_err = list(perr[4:3+N])+[np.sqrt(sum(perr[4:3+N]**2))] total_counts_fit_err = amplitude_fit_err - fit = self.spectrum_func(bins_keV,*params) - - line_pos_Hz_fit , line_pos_Hz_fit_err = ComplexLineShapeUtilities.energy_guess_to_frequency(line_pos_keV_fit, line_pos_keV_fit_err, self.B_field) - B_field_fit , B_field_fit_err = ComplexLineShapeUtilities.central_frequency_to_B_field(line_pos_Hz_fit, line_pos_Hz_fit_err) - fit_Hz = ComplexLineShapeUtilities.flip_array(fit) - bins_keV = bins_keV - line_pos_keV_fit + Constants.kr_k_line_e()/1000 - FWHM_eV_fit = FWHM_G_eV_fit - FWHM_eV_fit_err = FWHM_eV_G_fit_err + fit_Hz = self.spectrum_func(bins_Hz, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) nonzero_bins_index = np.where(data_hist_freq != 0)[0] zero_bins_index = np.where(data_hist_freq == 0)[0] @@ -398,9 +388,7 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): output_string += '-----------------\n' output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) output_string += '-----------------\n' - output_string += 'Gaussian FWHM = '+str(round(FWHM_G_eV_fit,2))+' +/- '+str(round(FWHM_eV_G_fit_err,2))+' eV\n' - output_string += '-----------------\n' - output_string += 'Line position \n= '+str(round(line_pos_Hz_fit,2))+' +/- '+str(round(line_pos_Hz_fit_err,2))+' Hz\n' + output_string += 'Gaussian FWHM = '+str(round(FWHM_eV_fit,2))+' +/- '+str(round(FWHM_eV_fit_err,2))+' eV\n' output_string += '-----------------\n' output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' output_string += '-----------------\n' @@ -416,15 +404,13 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): 'output_string': output_string, 'cov': cov, 'bins_keV': bins_keV, - 'fit': fit, + 'fit': fit_keV, 'bins_Hz': bins_Hz, 'fit_Hz': fit_Hz, - 'FWHM_eV_fit': FWHM_eV_fit, - 'FWHM_eV_fit_err': FWHM_eV_fit_err, - 'line_pos_Hz_fit': line_pos_Hz_fit, - 'line_pos_Hz_fit_err': line_pos_Hz_fit_err, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, + 'FWHM_eV_fit': FWHM_eV_fit, + 'FWHM_eV_fit_err': FWHM_eV_fit_err, 'prob_parameter_fit': prob_parameter_fit, 'prob_parameter_fit_err': prob_parameter_fit_err, 'scatter_proportion_fit': scatter_proportion_fit, @@ -472,24 +458,23 @@ def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak='shake'): current_full_spectrum += coefficient*current_working_spectrum return current_full_spectrum - def spectrum_func_1(self, x_keV, *p0): - x_eV = x_keV*1000. + def spectrum_func_1(self, bins_Hz, *p0): + B_field = p0[0] + FWHM_G_eV = p0[1] + amplitude = p0[2] + prob_parameter = p0[3] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) en_loss_array = self.std_eV_array() en_loss_array_min = en_loss_array[0] en_loss_array_max = en_loss_array[len(en_loss_array)-1] en_array_rev = ComplexLineShapeUtilities.flip_array(-1*en_loss_array) - f = np.zeros(len(x_keV)) - f_intermediate = np.zeros(len(x_keV)) + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) - FWHM_G_eV = p0[0] - line_pos_keV = p0[1] - amplitude = p0[2] - prob_parameter = p0[3] - - line_pos_eV = line_pos_keV*1000. - x_eV_minus_line = x_eV - line_pos_eV + x_eV_minus_line = Constants.kr_k_line_e() - x_eV zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] - nonzero_idx = [i for i in range(len(x_keV)) if i not in zero_idx] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] full_spectrum = self.make_spectrum_1(FWHM_G_eV, prob_parameter,) full_spectrum_rev = ComplexLineShapeUtilities.flip_array(full_spectrum) @@ -502,54 +487,46 @@ def fit_data_1(self, freq_bins, data_hist_freq): self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - bins_keV = ConversionFunctions.Energy(bins_Hz, self.B_field)/1000 - bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) - data_hist = ComplexLineShapeUtilities.flip_array(data_hist_freq) - #data_hist_err = ComplexLineShapeUtilities.get_hist_err_bins(data_hist) - bins_keV_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_keV, data_hist) + bins_Hz_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_Hz, data_hist_freq) + # Initial guesses for curve_fit + FWHM_guess = 5 + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + prob_parameter_guess = 0.5 # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) FWHM_eV_min = 1e-5 - FWHM_eV_max = (bins_keV[len(bins_keV)-1] - bins_keV[0])*1000 - line_pos_keV_min = bins_keV[0] - line_pos_keV_max = bins_keV[len(bins_keV)-1] + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) - ConversionFunctions.Energy(bins_Hz[-1], B_field_guess) amplitude_min = 1e-5 - amplitude_max = np.sum(data_hist)*3 + amplitude_max = np.sum(data_hist_freq)*3 prob_parameter_min = 1e-5 prob_parameter_max = 1 - # Initial guesses for curve_fit - FWHM_guess = 5 - line_pos_guess = bins_keV[np.argmax(data_hist)] - amplitude_guess = np.sum(data_hist)/2 - prob_parameter_guess = 0.5 - p0_guess = [FWHM_guess, line_pos_guess, amplitude_guess, prob_parameter_guess] - p0_bounds = ([FWHM_eV_min, line_pos_keV_min, amplitude_min, prob_parameter_min], - [FWHM_eV_max, line_pos_keV_max, amplitude_max, prob_parameter_max]) + + p0_guess = [B_field_guess, FWHM_guess, amplitude_guess, prob_parameter_guess] + p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min], + [B_field_max, FWHM_eV_max, amplitude_max, prob_parameter_max]) # Actually do the fitting - params , cov = curve_fit(self.spectrum_func_1, bins_keV_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) + params , cov = curve_fit(self.spectrum_func_1, bins_Hz_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) # Name each of the resulting parameters and errors ################### Generalize to N Gases ########################### - FWHM_G_eV_fit = params[0] - line_pos_keV_fit = params[1] - #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + B_field_fit = params[0] + FWHM_eV_fit = params[1] amplitude_fit = params[2] prob_parameter_fit = params[3] total_counts_fit = amplitude_fit perr = np.sqrt(np.diag(cov)) - FWHM_eV_G_fit_err = perr[0] - line_pos_keV_fit_err = perr[1] + B_field_fit_err = perr[0] + FWHM_eV_fit_err = perr[1] amplitude_fit_err = perr[2] prob_parameter_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - fit = self.spectrum_func_1(bins_keV,*params) - - line_pos_Hz_fit , line_pos_Hz_fit_err = ComplexLineShapeUtilities.energy_guess_to_frequency(line_pos_keV_fit, line_pos_keV_fit_err, self.B_field) - B_field_fit , B_field_fit_err = ComplexLineShapeUtilities.central_frequency_to_B_field(line_pos_Hz_fit, line_pos_Hz_fit_err) - fit_Hz = ComplexLineShapeUtilities.flip_array(fit) - bins_keV = bins_keV - line_pos_keV_fit + Constants.kr_k_line_e()/1000 - FWHM_eV_fit = FWHM_G_eV_fit - FWHM_eV_fit_err = FWHM_eV_G_fit_err + fit_Hz = self.spectrum_func_1(bins_Hz, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) nonzero_bins_index = np.where(data_hist_freq != 0)[0] zero_bins_index = np.where(data_hist_freq == 0)[0] @@ -565,9 +542,7 @@ def fit_data_1(self, freq_bins, data_hist_freq): output_string += '-----------------\n' output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) output_string += '-----------------\n' - output_string += 'Gaussian FWHM = '+str(round(FWHM_G_eV_fit,2))+' +/- '+str(round(FWHM_eV_G_fit_err,2))+' eV\n' - output_string += '-----------------\n' - output_string += 'Line position \n= '+str(round(line_pos_Hz_fit,2))+' +/- '+str(round(line_pos_Hz_fit_err,2))+' Hz\n' + output_string += 'Gaussian FWHM = '+str(round(FWHM_eV_fit,2))+' +/- '+str(round(FWHM_eV_fit_err,2))+' eV\n' output_string += '-----------------\n' output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' output_string += '-----------------\n' @@ -579,15 +554,13 @@ def fit_data_1(self, freq_bins, data_hist_freq): 'output_string': output_string, 'cov': cov, 'bins_keV': bins_keV, - 'fit': fit, + 'fit_keV': fit_keV, 'bins_Hz': bins_Hz, 'fit_Hz': fit_Hz, - 'FWHM_eV_fit': FWHM_eV_fit, - 'FWHM_eV_fit_err': FWHM_eV_fit_err, - 'line_pos_Hz_fit': line_pos_Hz_fit, - 'line_pos_Hz_fit_err': line_pos_Hz_fit_err, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, + 'FWHM_eV_fit': FWHM_eV_fit, + 'FWHM_eV_fit_err': FWHM_eV_fit_err, 'prob_parameter_fit': prob_parameter_fit, 'prob_parameter_fit_err': prob_parameter_fit_err, 'amplitude_fit': amplitude_fit, diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index e4b3e904..599ce131 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -22,7 +22,7 @@ def test_complex_lineshape(self): reader_config = { "action": "read", - "filename": "/host/march_2020_kr_calibration_channel_b_merged.root", + "filename": "/host/rid000037179_merged.root", "object_type": "TMultiTrackEventData", "object_name": "multiTrackEvents:Event", "use_katydid": False, @@ -30,11 +30,11 @@ def test_complex_lineshape(self): } complexLineShape_config = { 'bins_choice': np.linspace(0e6, 90e6, 1000), - 'gases': ["H2","Kr"], + 'gases': ["H2", "Kr"], 'max_scatters': 20, - 'fix_scatter_proportion': True, + 'fix_scatter_proportion': False, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below - 'gas_scatter_proportion': [0.61, 0.39], + 'gas_scatter_proportion': [1], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -86,7 +86,7 @@ def test_complex_lineshape(self): plot_title += '\n with fixed scatter proportion \n {}'.format(str_gas_scatter_proportion) plt.title(plot_title) plt.tight_layout() - plt.savefig('/host/plots/fit_shallow_trap_7418_with_{}_gas_scattering.png'.format(len(complexLineShape_config['gases']))) + plt.savefig('/host/plots/fit_FTC_march_with_{}_gas_scattering.png'.format(len(complexLineShape_config['gases']))) if __name__ == '__main__': From 021156b429103c70fe567282289d9be875950140 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Thu, 10 Sep 2020 15:11:00 -0400 Subject: [PATCH 009/199] composite gaussian resolution function implemented --- .../misc/MultiGasComplexLineShape.py | 208 +++++++++++++++++- test_analysis/Complex_line_shape_fitter.py | 18 +- 2 files changed, 208 insertions(+), 18 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 42d4934e..20f026d9 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -27,6 +27,7 @@ from scipy import integrate , signal, interpolate from itertools import product from math import factorial +from iminuit import Minuit import os import time import sys @@ -51,9 +52,14 @@ def InternalConfigure(self, params): self.bins_choice = reader.read_param(params, 'bins_choice', []) self.gases = reader.read_param(params, 'gases', ["H2", "Kr", "He", "Ar"]) self.max_scatters = reader.read_param(params, 'max_scatters', 20) - self.fix_scatter_proportion = reader.read_param(params, 'fix_scatter_proportion', True) - if self.fix_scatter_proportion == True: + self.fixed_scatter_proportion = reader.read_param(params, 'fixed_scatter_proportion', True) + if self.fixed_scatter_proportion == True: self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) + self.fit_ftc = reader.read_param(params, 'fit_ftc', True) + if self.fit_ftc == True: + self.A_array = reader.read_param(params, 'A_array', []) + self.sigma_array = reader.read_param(params, 'sigma_array', []) + self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', False) # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown self.num_points_in_std_array = reader.read_param(params, 'num_points_in_std_array', 10000) @@ -82,13 +88,16 @@ def InternalRun(self): # fit with shake spectrum data_hist_freq, freq_bins= np.histogram(a,bins=self.bins_choice) - histogram = data_hist_freq - bins = freq_bins - guess = np.where(np.array(histogram) == np.max(histogram))[0][0] - kr17kev_in_hz = guess*(bins[1]-bins[0])+bins[0] + # histogram = data_hist_freq +# bins = freq_bins +# guess = np.where(np.array(histogram) == np.max(histogram))[0][0] +# kr17kev_in_hz = guess*(bins[1]-bins[0])+bins[0] #self.B_field = B(17.8, kr17kev_in_hz + 0) - if self.fix_scatter_proportion == True: - self.results = self.fit_data_1(freq_bins, data_hist_freq) + if self.fixed_scatter_proportion == True: + if self.fit_ftc == True: + self.results = self.fit_data_ftc(freq_bins, data_hist_freq) + else: + self.results = self.fit_data_1(freq_bins, data_hist_freq) else: self.results = self.fit_data(freq_bins, data_hist_freq) @@ -109,12 +118,25 @@ def std_lorenztian_17keV(self): x_array = self.std_eV_array() ans = lorentzian(x_array,0,kr_line_width) return ans + + # A gaussian function + def gaussian(self, x_array, A, sigma, mu): + f = A*(1./(sigma*np.sqrt(2*np.pi)))*np.exp(-(((x_array-mu)/sigma)**2.)/2.) + return f # A gaussian centered at 0 eV with variable width, on the SELA def std_gaussian(self, sigma): x_array = self.std_eV_array() ans = ComplexLineShapeUtilities.gaussian(x_array,1,sigma,0) return ans + + def composite_gaussian(self, A_array, sigma_array): + x_array = self.std_eV_array() + ans = 0 + for A, sigma in zip(A_array, sigma_array): + ans += self.gaussian(x_array, A, sigma, 0) + return ans + # normalizes a function, but depends on binning. # Only to be used for functions evaluated on the SELA def normalize(self, f): @@ -177,14 +199,16 @@ def generate_scatter_convolution_file(self): f_radiation_loss = self.radiation_loss_f() scatter_spectra_single_gas[gas_type] = {} first_scatter = self.single_scatter_f(gas_type) - first_scatter = self.normalize(signal.convolve(first_scatter, f_radiation_loss, mode = 'same')) + if self.use_radiation_loss == True: + first_scatter = self.normalize(signal.convolve(first_scatter, f_radiation_loss, mode = 'same')) scatter_num_array = range(2, self.max_scatters+1) current_scatter = first_scatter scatter_spectra_single_gas[gas_type][str(1).zfill(2)] = current_scatter # x = std_eV_array() # diagnostic for i in scatter_num_array: current_scatter = self.another_scatter(current_scatter, gas_type) - current_scatter = self.normalize(signal.convolve(current_scatter, f_radiation_loss, mode = 'same')) + if self.use_radiation_loss == True: + current_scatter = self.normalize(signal.convolve(current_scatter, f_radiation_loss, mode = 'same')) scatter_spectra_single_gas[gas_type][str(i).zfill(2)] = current_scatter N = len(self.gases) scatter_spectra = {} @@ -249,6 +273,26 @@ def convolve_gaussian(self, func_to_convolve,gauss_FWHM_eV): ans = signal.convolve(resolution_f,func_to_convolve,mode='same') ans_normed = self.normalize(ans) return ans_normed + + def convolve_composite_gaussian(self, func_to_convolve, A_array, sigma_array): + resolution_f = self.composite_gaussian(A_array, sigma_array) + ans = signal.convolve(resolution_f, func_to_convolve, mode='same') + ans_normed = self.normalize(ans) + return ans_normed + + def least_square(self, bin_centers, hist, params): + # expectation + expectation = self.spectrum_func_ftc(bin_centers, *params) + + high_count_index = np.where(hist>0) + #low_count_index = np.where((hist>0) & (hist<=50)) + zero_count_index = np.where(hist==0) + + lsq = ((hist[high_count_index]- expectation[high_count_index])**2/hist[high_count_index]).sum() + #lsq += ((hist[low_count_index]- expectation[low_count_index])**2/hist[low_count_index]**2).sum() + #lsq += ((hist[zero_count_index]- expectation[zero_count_index])**2).sum() + + return lsq def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitted_peak='shake'): gases = self.gases @@ -568,4 +612,148 @@ def fit_data_1(self, freq_bins, data_hist_freq): 'data_hist_freq': data_hist_freq, 'reduced_chi2': reduced_chi2 } + return dictionary_of_fit_results + + def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): + gases = self.gases + current_path = self.path_to_scatter_spectra_file + # check_existence_of_scatter_files() + #filenames = list_files('scatter_spectra_files') + p = np.zeros(len(gases)) + p = self.scatter_proportion + scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') + scatter_spectra = np.load( + scatter_spectra_file_path, allow_pickle = True + ) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_composite_gaussian(current_working_spectrum, self.A_array, self.sigma_array) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += current_working_spectrum + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(len(self.gases))): + coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M + current_full_spectrum += coefficient*current_working_spectrum + return current_full_spectrum + + def spectrum_func_ftc(self, bins_Hz, *p0): + B_field = p0[0] + amplitude = p0[1] + prob_parameter = p0[2] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + en_array_rev = ComplexLineShapeUtilities.flip_array(-1*en_loss_array) + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_ftc(prob_parameter) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum) + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + return f + + def fit_data_ftc(self, freq_bins, data_hist_freq): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + bins_Hz_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_Hz, data_hist_freq) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + prob_parameter_guess = 0.5 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max)] + logger.info(p0_guess) + logger.info(p0_bounds) + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.least_square(bins_Hz, data_hist_freq, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + # Name each of the resulting parameters and errors + ################### Generalize to N Gases ########################### + B_field_fit = params[0] + amplitude_fit = params[1] + prob_parameter_fit = params[2] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + prob_parameter_fit_err = perr[2] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_ftc(bins_Hz, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + + nonzero_bins_index = np.where(data_hist_freq != 0)[0] + zero_bins_index = np.where(data_hist_freq == 0)[0] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_zero = fit_Hz[zero_bins_index] + data_Hz_zero = data_hist_freq[zero_bins_index] + chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) + reduced_chi2 = chi2/(len(data_hist_freq)-4) + elapsed = time.time() - t + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)\ + +' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' + output_string += '-----------------\n' + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'prob_parameter_fit': prob_parameter_fit, + 'prob_parameter_fit_err': prob_parameter_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } return dictionary_of_fit_results \ No newline at end of file diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 599ce131..4933c108 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -7,7 +7,6 @@ import numpy as np import unittest import matplotlib.pyplot as plt -from root_numpy import tree2array import ROOT as r from morpho.utilities import morphologging, parser @@ -22,7 +21,7 @@ def test_complex_lineshape(self): reader_config = { "action": "read", - "filename": "/host/rid000037179_merged.root", + "filename": "/host/march_2020_kr_calibration_channel_b_merged.root", "object_type": "TMultiTrackEventData", "object_name": "multiTrackEvents:Event", "use_katydid": False, @@ -30,11 +29,14 @@ def test_complex_lineshape(self): } complexLineShape_config = { 'bins_choice': np.linspace(0e6, 90e6, 1000), - 'gases': ["H2", "Kr"], + 'gases': ["H2", "Ar"], 'max_scatters': 20, - 'fix_scatter_proportion': False, + 'fixed_scatter_proportion': True, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below - 'gas_scatter_proportion': [1], + 'gas_scatter_proportion': [0.93, 0.02], + 'fit_ftc': True, + 'A_array': [0.076, 0.341, 0.381, 0.203], + 'sigma_array': [5.01, 13.33, 15.40, 11.85], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -75,8 +77,8 @@ def test_complex_lineshape(self): plt.plot(results['bins_Hz']/1e9, results['fit_Hz'], label = results['output_string'], alpha = 0.7) plt.legend(loc = 'upper left', fontsize = 12) plt.xlabel('frequency GHz') - plot_title = 'fit shallow trap 7418 with {} gas scattering'.format(len(complexLineShape_config['gases'])) - if complexLineShape_config['fix_scatter_proportion'] == True: + plot_title = 'fit ftc oct with {} gas scattering'.format(len(complexLineShape_config['gases'])) + if complexLineShape_config['fixed_scatter_proportion'] == True: str_gas_scatter_proportion = '' for i in range(len(complexLineShape_config['gases'])): str_gas_scatter_proportion += complexLineShape_config['gases'][i] @@ -86,7 +88,7 @@ def test_complex_lineshape(self): plot_title += '\n with fixed scatter proportion \n {}'.format(str_gas_scatter_proportion) plt.title(plot_title) plt.tight_layout() - plt.savefig('/host/plots/fit_FTC_march_with_{}_gas_scattering.png'.format(len(complexLineShape_config['gases']))) + plt.savefig('/host/plots/fit_FTC_oct_with_{}_gas_scattering.png'.format(len(complexLineShape_config['gases']))) if __name__ == '__main__': From 6727ea2a6ca4668e19d8cfb37434acfbb8dc94cd Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Tue, 15 Sep 2020 10:37:39 -0400 Subject: [PATCH 010/199] add poisson chi2 --- .../misc/MultiGasComplexLineShape.py | 54 +++++++++++++++---- test_analysis/Complex_line_shape_fitter.py | 2 +- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 20f026d9..3f5736f7 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -68,7 +68,8 @@ def InternalConfigure(self, params): self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') - self.path_to_missing_track_radiation_loss_data_numpy_file = '/host' + self.path_to_missing_track_radiation_loss_data_numpy_file = '/host/' + self.path_to_ins_resolution_data_txt = '/host/' if not os.path.exists(self.shake_spectrum_parameters_json_path): raise IOError('Shake spectrum path does not exist') @@ -279,7 +280,25 @@ def convolve_composite_gaussian(self, func_to_convolve, A_array, sigma_array): ans = signal.convolve(resolution_f, func_to_convolve, mode='same') ans_normed = self.normalize(ans) return ans_normed - + + def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): + ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt+'ins_resolution_all.txt') + x_data = ins_resolution_data.T[0] + y_data = ins_resolution_data.T[1] + y_err_data = ins_resolution_data.T[2] + return x_data, y_data, y_err_data + + def convolve_ins_resolution(self, working_spectrum): + x_data, y_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + f = interpolate.interp1d(x_data, y_data) + x_array = self.std_eV_array() + y_array = np.zeros(len(x_array)) + index_within_range_of_xdata = np.where((x_array >= x_data[0]) & (x_array <= x_data[-1])) + y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]) + convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') + normalized_convolved_spectrum = self.normalize(convolved_spectrum) + return normalized_convolved_spectrum + def least_square(self, bin_centers, hist, params): # expectation expectation = self.spectrum_func_ftc(bin_centers, *params) @@ -294,6 +313,26 @@ def least_square(self, bin_centers, hist, params): return lsq + def reduced_chi2_Pearson_Neyman_composite(self, data_hist_freq, fit_Hz, number_of_parameters): + nonzero_bins_index = np.where(data_hist_freq != 0)[0] + zero_bins_index = np.where(data_hist_freq == 0)[0] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_zero = fit_Hz[zero_bins_index] + data_Hz_zero = data_hist_freq[zero_bins_index] + chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) + reduced_chi2 = chi2/(len(data_hist_freq) - number_of_parameters) + return reduced_chi2 + + # following the expression in the paper Steve BAKER and Robert D. COUSINS, (1984) CLARIFICATION OF THE USE OF CHI-SQUARE AND LIKELIHOOD FUNCTIONS IN FITS TO HISTOGRAMS + def reduced_chi2_Poisson(self, data_hist_freq, fit_Hz, number_of_parameters): + nonzero_bins_index = np.where(data_hist_freq != 0) + chi2 = ((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + logger.info(chi2) + logger.info(len(data_hist_freq) - number_of_parameters) + reduced_chi2 = chi2/(len(data_hist_freq) - number_of_parameters) + return reduced_chi2 + def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitted_peak='shake'): gases = self.gases max_scatters = self.max_scatters @@ -631,7 +670,7 @@ def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() - current_working_spectrum = self.convolve_composite_gaussian(current_working_spectrum, self.A_array, self.sigma_array) + current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum N = len(self.gases) @@ -720,14 +759,7 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) - nonzero_bins_index = np.where(data_hist_freq != 0)[0] - zero_bins_index = np.where(data_hist_freq == 0)[0] - fit_Hz_nonzero = fit_Hz[nonzero_bins_index] - data_Hz_nonzero = data_hist_freq[nonzero_bins_index] - fit_Hz_zero = fit_Hz[zero_bins_index] - data_Hz_zero = data_hist_freq[zero_bins_index] - chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) - reduced_chi2 = chi2/(len(data_hist_freq)-4) + reduced_chi2 = self.reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = 4) elapsed = time.time() - t output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 4933c108..29053a7e 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -88,7 +88,7 @@ def test_complex_lineshape(self): plot_title += '\n with fixed scatter proportion \n {}'.format(str_gas_scatter_proportion) plt.title(plot_title) plt.tight_layout() - plt.savefig('/host/plots/fit_FTC_oct_with_{}_gas_scattering.png'.format(len(complexLineShape_config['gases']))) + plt.savefig('/host/plots/fit_FTC_march_with_{}_gas_scattering.png'.format(len(complexLineShape_config['gases']))) if __name__ == '__main__': From 5a8fae729a9131d3283011d292d880e5c8fa27bf Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Tue, 15 Sep 2020 20:06:21 -0400 Subject: [PATCH 011/199] update KrComplexLineShape to use simulated resolution for fake data generator --- .../processors/misc/KrComplexLineShape.py | 25 ++++++++++++++++--- .../misc/MultiGasComplexLineShape.py | 22 ++++++++++------ test_analysis/Complex_line_shape_fitter.py | 2 +- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/mermithid/processors/misc/KrComplexLineShape.py b/mermithid/processors/misc/KrComplexLineShape.py index bba80328..53500dca 100644 --- a/mermithid/processors/misc/KrComplexLineShape.py +++ b/mermithid/processors/misc/KrComplexLineShape.py @@ -60,7 +60,8 @@ def InternalConfigure(self, params): self.B_field = reader.read_param(params, 'B_field', 0.957810722501) self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') - + self.path_to_ins_resolution_data_txt = '/host/' + if not os.path.exists(self.shake_spectrum_parameters_json_path): raise IOError('Shake spectrum path does not exist') if not os.path.exists(self.path_to_osc_strengths_files): @@ -231,7 +232,25 @@ def convolve_gaussian(self, func_to_convolve,gauss_FWHM_eV): ans_normed = self.normalize(ans) return ans_normed - def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitted_peak='shake'): + def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): + ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt+'ins_resolution_all.txt') + x_data = ins_resolution_data.T[0] + y_data = ins_resolution_data.T[1] + y_err_data = ins_resolution_data.T[2] + return x_data, y_data, y_err_data + + def convolve_ins_resolution(self, working_spectrum): + x_data, y_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + f = interpolate.interp1d(x_data, y_data) + x_array = self.std_eV_array() + y_array = np.zeros(len(x_array)) + index_within_range_of_xdata = np.where((x_array >= x_data[0]) & (x_array <= x_data[-1])) + y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]) + convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') + normalized_convolved_spectrum = self.normalize(convolved_spectrum) + return normalized_convolved_spectrum + + def make_spectrum(self, prob_parameter, scatter_proportion, emitted_peak='shake'): gases = self.gases max_scatters = self.max_scatters max_comprehensive_scatters = self.max_comprehensive_scatters @@ -249,7 +268,7 @@ def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitt current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() - current_working_spectrum = self.convolve_gaussian(current_working_spectrum, gauss_FWHM_eV) + current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum for n in range(1, max_comprehensive_scatters + 1): diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 3f5736f7..2db86a03 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -308,11 +308,19 @@ def least_square(self, bin_centers, hist, params): zero_count_index = np.where(hist==0) lsq = ((hist[high_count_index]- expectation[high_count_index])**2/hist[high_count_index]).sum() - #lsq += ((hist[low_count_index]- expectation[low_count_index])**2/hist[low_count_index]**2).sum() - #lsq += ((hist[zero_count_index]- expectation[zero_count_index])**2).sum() - + #lsq += ((hist[low_count_index]- expectation[low_count_index])**2/hist[low_count_index]).sum() + lsq += ((hist[zero_count_index]- expectation[zero_count_index])**2).sum() return lsq + def chi2_Poisson(self, bin_centers, data_hist_freq, params): + nonzero_bins_index = np.where(data_hist_freq != 0) + zero_bins_index = np.where(data_hist_freq == 0) + # expectation + fit_Hz = self.spectrum_func_ftc(bin_centers, *params) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + return chi2 + def reduced_chi2_Pearson_Neyman_composite(self, data_hist_freq, fit_Hz, number_of_parameters): nonzero_bins_index = np.where(data_hist_freq != 0)[0] zero_bins_index = np.where(data_hist_freq == 0)[0] @@ -327,9 +335,9 @@ def reduced_chi2_Pearson_Neyman_composite(self, data_hist_freq, fit_Hz, number_o # following the expression in the paper Steve BAKER and Robert D. COUSINS, (1984) CLARIFICATION OF THE USE OF CHI-SQUARE AND LIKELIHOOD FUNCTIONS IN FITS TO HISTOGRAMS def reduced_chi2_Poisson(self, data_hist_freq, fit_Hz, number_of_parameters): nonzero_bins_index = np.where(data_hist_freq != 0) - chi2 = ((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() - logger.info(chi2) - logger.info(len(data_hist_freq) - number_of_parameters) + zero_bins_index = np.where(data_hist_freq == 0) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() reduced_chi2 = chi2/(len(data_hist_freq) - number_of_parameters) return reduced_chi2 @@ -734,7 +742,7 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): logger.info(p0_guess) logger.info(p0_bounds) # Actually do the fitting - m_binned = Minuit.from_array_func(lambda p: self.least_square(bins_Hz, data_hist_freq, p), + m_binned = Minuit.from_array_func(lambda p: self.chi2_Poisson(bins_Hz, data_hist_freq, p), start = p0_guess, limit = p0_bounds, throw_nan = True diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 29053a7e..8d5db099 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -29,7 +29,7 @@ def test_complex_lineshape(self): } complexLineShape_config = { 'bins_choice': np.linspace(0e6, 90e6, 1000), - 'gases': ["H2", "Ar"], + 'gases': ["H2", "He"], 'max_scatters': 20, 'fixed_scatter_proportion': True, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below From 69617b906b2e84e63812796e6d0b22551ffcb5fd Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 16 Sep 2020 22:27:22 -0400 Subject: [PATCH 012/199] add convolve_ins_resolution --- .../processors/misc/KrComplexLineShape.py | 49 ++++++++++++------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/mermithid/processors/misc/KrComplexLineShape.py b/mermithid/processors/misc/KrComplexLineShape.py index b273ad90..3624efab 100644 --- a/mermithid/processors/misc/KrComplexLineShape.py +++ b/mermithid/processors/misc/KrComplexLineShape.py @@ -250,8 +250,26 @@ def convolve_gaussian(self, func_to_convolve,gauss_FWHM_eV): ans = signal.convolve(resolution_f,func_to_convolve,mode='same') ans_normed = self.normalize(ans) return ans_normed - - def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitted_peak='shake'): + + def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): + ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt+'ins_resolution_all4.txt') + x_data = ins_resolution_data.T[0] + y_data = ins_resolution_data.T[1] + y_err_data = ins_resolution_data.T[2] + return x_data, y_data, y_err_data + + def convolve_ins_resolution(self, working_spectrum): + x_data, y_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + f = interpolate.interp1d(x_data, y_data) + x_array = self.std_eV_array() + y_array = np.zeros(len(x_array)) + index_within_range_of_xdata = np.where((x_array >= x_data[0]) & (x_array <= x_data[-1])) + y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]) + convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') + normalized_convolved_spectrum = self.normalize(convolved_spectrum) + return normalized_convolved_spectrum + + def make_spectrum(self, prob_parameter, scatter_proportion, emitted_peak='shake'): gases = self.gases max_scatters = self.max_scatters max_comprehensive_scatters = self.max_comprehensive_scatters @@ -271,7 +289,7 @@ def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitt current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() elif emitted_peak == 'dirac': current_working_spectrum = self.std_dirac() - current_working_spectrum = self.convolve_gaussian(current_working_spectrum, gauss_FWHM_eV) + current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum for n in range(1, max_comprehensive_scatters + 1): @@ -312,18 +330,17 @@ def spectrum_func(self, x_keV, *p0): f = np.zeros(len(x_keV)) f_intermediate = np.zeros(len(x_keV)) - FWHM_G_eV = p0[0] - line_pos_keV = p0[1] - amplitude = p0[2] - prob_parameter = p0[3] - scatter_proportion = p0[4] + line_pos_keV = p0[0] + amplitude = p0[1] + prob_parameter = p0[2] + scatter_proportion = p0[3] line_pos_eV = line_pos_keV*1000. x_eV_minus_line = x_eV - line_pos_eV zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_keV)) if i not in zero_idx] - full_spectrum = self.make_spectrum(FWHM_G_eV, prob_parameter, scatter_proportion) + full_spectrum = self.make_spectrum(prob_parameter, scatter_proportion) full_spectrum_rev = ComplexLineShapeUtilities.flip_array(full_spectrum) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum_rev) f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) @@ -358,7 +375,6 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): scatter_proportion_min = 1e-5 scatter_proportion_max = 1 # Initial guesses for curve_fit - FWHM_guess = 5 line_pos_guess = bins_keV[np.argmax(data_hist)] amplitude_guess = np.sum(data_hist)/2 prob_parameter_guess = 0.5 @@ -434,7 +450,7 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): } return dictionary_of_fit_results - def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak='shake'): + def make_spectrum_1(self, prob_parameter, emitted_peak='shake'): gases = self.gases max_scatters = self.max_scatters max_comprehensive_scatters = self.max_comprehensive_scatters @@ -454,7 +470,7 @@ def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak='shake'): current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() elif emitted_peak == 'dirac': current_working_spectrum = self.std_dirac() - current_working_spectrum = self.convolve_gaussian(current_working_spectrum, gauss_FWHM_eV) + current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum for n in range(1, max_comprehensive_scatters + 1): @@ -492,17 +508,16 @@ def spectrum_func_1(self, x_keV, *p0): f = np.zeros(len(x_keV)) f_intermediate = np.zeros(len(x_keV)) - FWHM_G_eV = p0[0] - line_pos_keV = p0[1] - amplitude = p0[2] - prob_parameter = p0[3] + line_pos_keV = p0[0] + amplitude = p0[1] + prob_parameter = p0[2] line_pos_eV = line_pos_keV*1000. x_eV_minus_line = x_eV - line_pos_eV zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_keV)) if i not in zero_idx] - full_spectrum = self.make_spectrum_1(FWHM_G_eV, prob_parameter,emitted_peak=self.base_shape) + full_spectrum = self.make_spectrum_1(prob_parameter, emitted_peak=self.base_shape) full_spectrum_rev = ComplexLineShapeUtilities.flip_array(full_spectrum) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum_rev) f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) From 6ac5ac17dfbe707f0c636e44c5c4c9e6526d8502 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Tue, 22 Sep 2020 22:51:06 -0400 Subject: [PATCH 013/199] resolve probabilities containing NaN --- mermithid/misc/FakeTritiumDataFunctions.py | 4 ++-- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 3 ++- mermithid/processors/misc/KrComplexLineShape.py | 1 + test_analysis/fake_data_stan_analysis.py | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 68308718..a8c6cb5e 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -263,7 +263,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., ls_params[0], 0, 1, ls_params[1]) + lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., 0, 1, ls_params[1]) beta_rates = np.zeros(len(K)) for i,ke in enumerate(K): @@ -294,7 +294,7 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., ls_params[0], 0, 1, ls_params[1]) + lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., 0, 1, ls_params[1]) bkgd_rates = np.full(len(K), bkgd_rate()) if len(K) < len(K_lineshape): diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index c6c122f3..f391579d 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -17,6 +17,7 @@ from morpho.processors import BaseProcessor from mermithid.misc.FakeTritiumDataFunctions import * from mermithid.processors.misc.KrComplexLineShape import KrComplexLineShape +from mermithid.misc import Constants, ComplexLineShapeUtilities, ConversionFunctions logger = morphologging.getLogger(__name__) @@ -101,7 +102,7 @@ def InternalConfigure(self, params): #paths self.simplified_scattering_path = reader.read_param(params, 'simplified_scattering_path', '/host/input_data/simplified_scattering_params.txt') - self.detailed_scatter_spectra_path = reader.read_param(params, 'path_to_detailed_scatter_spectra_dir', '.') + self.detailed_scatter_spectra_path = reader.read_param(params, 'path_to_detailed_scatter_spectra_dir', '/host') self.efficiency_path = reader.read_param(params, 'efficiency_path', '') #options diff --git a/mermithid/processors/misc/KrComplexLineShape.py b/mermithid/processors/misc/KrComplexLineShape.py index 3624efab..eabc5383 100644 --- a/mermithid/processors/misc/KrComplexLineShape.py +++ b/mermithid/processors/misc/KrComplexLineShape.py @@ -63,6 +63,7 @@ def InternalConfigure(self, params): self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') self.base_shape = reader.read_param(params, 'base_shape', 'shake') self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') + self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/') if self.base_shape=='shake' and not os.path.exists(self.shake_spectrum_parameters_json_path): raise IOError('Shake spectrum path does not exist') diff --git a/test_analysis/fake_data_stan_analysis.py b/test_analysis/fake_data_stan_analysis.py index 00ebdce0..0ee91e6b 100644 --- a/test_analysis/fake_data_stan_analysis.py +++ b/test_analysis/fake_data_stan_analysis.py @@ -112,7 +112,7 @@ def GenerateFakeData(inputs_dict): """ specGen_config = { "apply_efficiency": True, - "efficiency_path": "../phase2_detection_efficiency_curve/combined_energy_corrected_count_rates/combined_energy_corrected_eff_at_quad_trap_frequencies.json", + "efficiency_path": "../tests/combined_energy_corrected_eff_at_quad_trap_frequencies.json", "detailed_or_simplified_lineshape": "detailed", "return_frequency": True, "Q": inputs_dict["Q"], @@ -154,7 +154,7 @@ def BinAndSaveData(tritium_data, nbins, root_file="./results/tritium_analysis.ro "energy_or_frequency": 'frequency', "variables": "F", "title": "corrected_spectrum", - "efficiency_filepath": "../phase2_detection_efficiency_curve/combined_energy_corrected_count_rates/combined_energy_corrected_eff_at_quad_trap_frequencies.json", + "efficiency_filepath": "../tests/combined_energy_corrected_eff_at_quad_trap_frequencies.json", 'bins': np.linspace(tritium_data['minf'], tritium_data['maxf'], nbins), 'fss_bins': False # If fss_bins is True, bins is ignored and overridden } From d5d090acc3175f60f2297f10f47e3114e7487509 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Sun, 27 Sep 2020 07:56:05 -0400 Subject: [PATCH 014/199] testing stan analysis test scripts in termite --- .../fake_data_stan_analysis_termite.py | 513 ++++++++++++++++++ 1 file changed, 513 insertions(+) create mode 100644 test_analysis/fake_data_stan_analysis_termite.py diff --git a/test_analysis/fake_data_stan_analysis_termite.py b/test_analysis/fake_data_stan_analysis_termite.py new file mode 100644 index 00000000..36db704b --- /dev/null +++ b/test_analysis/fake_data_stan_analysis_termite.py @@ -0,0 +1,513 @@ +# +# fake_data_stan_analysis.py +# Author: T. E. Weiss +# Date modified: June 2, 2020 +# +# This script generates fake data, then analyzes the data in Stan to infer posteriors. +# Pathnames configured for running from: termite/phase2_main_scripts/ +# + +""" +To-do: + - In FakeExperimentEnsemble, add: + 1. Tracking of convergence issues, so that a summary of problems can be saved/printed + 2. An option to parallelize with slurm instead of multiprocessing + - Run morpho processor for ensemble-analysis plotting, once it has been tested sufficiently +""" + + +#import unittest +from morpho.utilities import morphologging +logger = morphologging.getLogger(__name__) +import logging +import numpy as np +import time +import argparse +parser = argparse.ArgumentParser() + +from scipy.interpolate import interp1d +import json + + +#Importing constants from mermithid +from mermithid.misc.Constants import * +from mermithid.processors.TritiumSpectrum.FakeDataGenerator import FakeDataGenerator +from mermithid.processors.misc.TritiumAndEfficiencyBinner import TritiumAndEfficiencyBinner +#Importing processors from morpho +from morpho.processors.sampling import PyStanSamplingProcessor, PriorSamplingProcessor +from morpho.processors.plots import Histogram, APosterioriDistribution, Histo2dDivergence +from morpho.processors.IO import IOROOTProcessor, IORProcessor + +#Defining processors +priorSampler = PriorSamplingProcessor("sample") +specGen = FakeDataGenerator("specGen") +writerProcessor = IOROOTProcessor("writer") +rReaderProcessor = IORProcessor("reader") +analysisProcessor = PyStanSamplingProcessor("analyzer") +histPlotter = Histogram("histo") +aposterioriPlotter = APosterioriDistribution("posterioriDistrib") +divPlotter = Histo2dDivergence("2dDivergence") + + + +def DefineGeneratorInputs(root_file='./results/tritium_analysis.root'): + """ + Samples inputs to a fake data generator from priors, then combines them in a dictionary with fixed inputs to the generator. Saves all the inputs to a root file. + + Returns: + generator_inputs: dictionary of all inputs to a fake data generator + """ + prior_sampler_config = { + "fixed_inputs": { + 'Nscatters': const_dict['Nscatters_generation'], + 'minf': const_dict['minf'], #In Hz + 'err_from_B': const_dict['err_from_B_generation'], + 'H2_scatter_prop': const_dict['H2_scatter_prop_tritium'], + }, + "priors": [ + {'name': 'Q', 'prior_dist': 'normal', 'prior_params': const_dict['Q']}, + {'name': 'mass', 'prior_dist': 'gamma', 'prior_params': const_dict['mass']}, + {'name': 'sigma', 'prior_dist': 'normal', 'prior_params': const_dict['sigma']}, #From Ali's complex lineshape fits. Final states also included + {'name': 'S', 'prior_dist': 'poisson', 'prior_params': const_dict['S']}, + {'name': 'B_1kev', 'prior_dist': 'lognormal', 'prior_params': const_dict['B_1kev']}, + {'name': 'survival_prob', 'prior_dist': 'beta', 'prior_params': const_dict['survival_prob']}, #Centered around 0.736. To be replaced given complex lineshape result/systematic assessment + {'name': 'Bfield', 'prior_dist': 'normal', 'prior_params': const_dict['Bfield']}, #From complex lineshape fit to calibration data. More sig figs on mean needed? + ] + } + + inputs_writer_config = { + "action": "write", + "tree_name": "input", + "filename": root_file, + "variables": [ + {"variable": "Nscatters", "type":"int"}, + {"variable": "minf", "type": "float"}, + {"variable": "err_from_B", "type": "float"}, + {"variable": "Q", "type": "float"}, + {"variable": "mass", "type": "float"}, + {"variable": "sigma", "type": "float"}, + {"variable": "S", "type": "float"}, + {"variable": "B_1kev", "type": "float"}, + {"variable": "survival_prob", "type": "float"}, + {"variable": "Bfield", "type": "float"} + ]} + + #Configuration step + priorSampler.Configure(prior_sampler_config) + writerProcessor.Configure(inputs_writer_config) + + #Sampling inputs + priorSampler.Run() + generator_inputs = priorSampler.results + + #Saving results + gen_inputs_root = {key:[value] for key, value in generator_inputs.items()} + writerProcessor.data = gen_inputs_root + writerProcessor.Run() + + return generator_inputs + + +def GenerateFakeData(inputs_dict): + """ + Generates fake Phase II tritium beta spectrum data and plots it. + + Arguments: + - inputs_dict: dictionary of parameter values inputted to the fake data generator. + + Returns: + - results: dict with + 1) keys: K (energy), F (frequency) + 2) mapped values: energy, frequency + """ + specGen_config = { + "apply_efficiency": True, + "efficiency_path": "/host-termite/analysis_input/combined_energy_corrected_eff_at_quad_trap_frequencies.json", + "detailed_or_simplified_lineshape": "detailed", + "return_frequency": True, + "Q": inputs_dict["Q"], + "mass": inputs_dict["mass"], + "minf": inputs_dict["minf"], + "scattering_sigma": inputs_dict["sigma"], + "S": inputs_dict["S"], + "B_1kev": inputs_dict["B_1kev"], + "survival_prob": inputs_dict["survival_prob"], + "err_from_B": inputs_dict["err_from_B"], + "Nscatters": inputs_dict["Nscatters"], + "B_field": inputs_dict["Bfield"], + "scatter_proportion": inputs_dict["H2_scatter_prop"], + "n_steps": 100000, + } + + histo_config = { + "variables": "F", + "n_bins_x": 65, + "output_path": "./results/", + "title": "psuedo-data"+str(inputs_dict['Q']), + "format": "pdf" + } + + #Configuration step + specGen.Configure(specGen_config) + histPlotter.Configure(histo_config) + #Generate data + specGen.Run() + results = specGen.results + #Plot histograms of generated data + histPlotter.data = {'F':results['F'].tolist()} + histPlotter.Run() + + return results + + +def BinAndSaveData(tritium_data, nbins, root_file="./results/tritium_analysis.root"): + eff_path = "/host-termite/analysis_input/combined_energy_corrected_eff_at_quad_trap_frequencies.json" + + binner_config = { + "energy_or_frequency": 'frequency', + "variables": "F", + "title": "corrected_spectrum", + "efficiency_filepath": eff_path, + 'bins': np.linspace(tritium_data['minf'], tritium_data['maxf'], nbins), #(tritium_data['maxf']-tritium_data['minf'])/float(nbins) + 'fss_bins': False # If fss_bins is True, bins is ignored and overridden + } + + binner = TritiumAndEfficiencyBinner("binner") + binner.Configure(binner_config) + binner.data = tritium_data + binner.Run() + results = binner.results + + eff_means = results['bin_efficiencies'] + eff_errs = (results['bin_efficiency_errors'][0]+results['bin_efficiency_errors'][1])/2. + for i in range(len(eff_means)): + if (eff_means[i]K conversion + "Bfield_ctr": const_dict['Bfield'][0], #From complex lineshape fit to calibration + "Bfield_std": const_dict['Bfield'][1], #data. More sig figs on mean needed? + "survival_prob_alpha": const_dict['survival_prob'][0], #Centered around ~0.736. To be replaced + "survival_prob_beta": const_dict['survival_prob'][1], #given complex lineshape result+systematics + "Q_ctr": QT2(), + "Q_std": const_dict['Q_std_analysis'], + "m_alpha": const_dict['mass'][0], + "m_beta": 1./const_dict['mass'][1], + "B_1kev_logctr": const_dict['B_1kev'][0], + "B_1kev_logstd": const_dict['B_1kev'][1], + "KEscale": const_dict['KEscale'], #This enables the option of cmdstan running +# "slope": 0.000390369173, #For efficiency modeling with unbinned data +# "intercept": -6.00337656, + "Nscatters": const_dict['Nscatters_analysis'] #Because peaks>16 in simplified linesahpe have means->inf as FWHM->0 + }, + "interestParams": interest_params, + } + + + vars_to_save = [ + {"variable": "Q", "type": "float"}, + {"variable": "mass", "type": "float"}, + {"variable": "survival_prob", "type": "float"}, + {"variable": "Bfield", "type": "float"}, + {"variable": "sigma", "type": "float"}, + {"variable": "S", "type": "float"}, + {"variable": "B_1kev", "type": "float"}, + {"variable": "B", "type": "float"}, + {"variable": "KEmin", "type": "float"}, + {"variable": "Ndata_gen", "type": "float"}, + {"variable": "rate_param", "type": "float"}, + {"variable": "KE_sample", "type": "float"}, + {"variable": "Nfit_signal", "type": "float"}, + {"variable": "Nfit_bkgd", "type": "float"}, + {"variable": "divergent__", "root_alias": "divergence", "type": "float"}, + {"variable": "energy__", "root_alias": "energy", "type": "float"}, + {"variable": "lp_prob", "root_alias": "lp_prob", "type": "float"} + ] + + for i in range(Nbins): + vars_to_save.append({"variable": 'Nfit_bins[{}]'.format(str(i)), "root_alias": 'Nfit_bins{}'.format(str(i)), "type": "float"}) + + + posteriors_writer_config = { + "action": "write", + "tree_name": "analysis", + "file_option": "update", + "filename": root_file, + "variables": vars_to_save} + + #Configuration step + rReaderProcessor.Configure(scattering_reader_config) + analysisProcessor.Configure(analyzer_config) + writerProcessor.Configure(posteriors_writer_config) + + #Make data accessible to analyzer + analysisProcessor.data = tritium_data + analysisProcessor.data = {'Nbins': Nbins} + + #Make scattering parameters accessible to analyzer + rReaderProcessor.Run() + pi = rReaderProcessor.data + analysisProcessor.data = {key:val[:analysisProcessor.data['Nscatters']] for key, val in pi.items()} + + #Run analysis + analysisProcessor.Run() + results = analysisProcessor.results + + #Save results + writerProcessor.data = results + writerProcessor.Run() + + return results + + +def PlotStanResults(posteriors, correlation_vars=['Q', 'mass'], divergence_vars=['Q', 'Bfield', 'survival_prob']): + """ + Creates and saves two plots: + a) posteriors and correlations between them, and + b) plot showing where in parameter space Hamiltonian Monte Carlo divergences occured. + + Required argument: + 1) posteriors: dict; output of the PyStanSamplingProcessor + Optional arguments: + 2) correlation_vars: list of strings; names of variables to be included in correlation plot + 3) divergence_vars: list of strings; names of variables to be included in divergences plot + """ + aposteriori_config = { + "n_bins_x": 50, #Potentially increase + "n_bins_y": 50, + "variables": correlation_vars, + "title": "Q_vs_Kmin", + "output_path": "./plots" + } + + div_plot_config = { + "n_bins_x": 50, + "n_bins_y": 50, + "variables": divergence_vars, + "title": "div_plot", + "output_path": "./plots" + } + + #Configuration step + aposterioriPlotter.Configure(aposteriori_config) + divPlotter.Configure(div_plot_config) + + #Plots correlations between posteriors + aposterioriPlotter.data = posteriors + aposterioriPlotter.Run() + + #Plot 2D grid of divergent points + divPlotter.data = posteriors + divPlotter.Run() + + +def PerformFakeExperiment(root_filename, plot_results=False, parallelized=True, bin_data=True, wait=45): + """ + Generate fake tritium data and analyze it in Stan. If plot_results==True, create correlation and divergence plots. + + Saves generation inputs, generation results, and analysis results to different trees of one ROOT file. + """ + if parallelized==True: + flist = root_filename.split('/') + run_num = float(flist[len(flist)-1][0]) + time.sleep(run_num*wait) #Optionally stagger the runs slightly, either for debugging/clarity of output, or to avoid any possible memory overflows + logger.info("----------------------MORPHO RUN #{}----------------------".format(int(run_num))) + + + #Sample inputs to the data generator and save to one branch of a root file: + inputs_dict = DefineGeneratorInputs(root_filename) + + #Generate data using the inputs + tritium_data_unbinned = GenerateFakeData(inputs_dict) + + #Optionally bin data, then save it + if bin_data: + nbins = const_dict['f_nbins'] + tritium_data = BinAndSaveData(tritium_data_unbinned, nbins, root_filename) + else: + tritium_data = SaveUnbinnedData(tritium_data, root_filename) + + #Analyze data and save posteriors + posteriors = StanTritiumAnalysis(tritium_data, root_file=root_filename) + + #Optionally plot posteriors + if plot_results == True: + PlotStanResults(posteriors) + + +def CalibrateResults(root_filenames, vars_to_calibrate, cred_interval=[0.05, 0.95]): + from morpho.processors.diagnostics.CalibrationProcessor import CalibrationProcessor + calibrator = CalibrationProcessor("calib") + + calib_config = { + "files": root_filenames, + "in_param_names": vars_to_calibrate, + "cred_interval": cred_interval, + "quantiles": True + } + #Configuration step + check_success = calibrator.Configure(calib_config) + if check_success == False: + return + + calibrator.Run() + + +def FakeExperimentEnsemble(n_runs, root_basename, parallelize=True, n_processes=4, vars_to_calibrate=['Q']): + """ + To-do: add parallelization option for a Slurm environment. + + n_runs: int; number of pseudo-experiments to be performed + root_basename: str; results are saved in rootfiles labeled by root_basename and the pseudo-experiment number + """ + if n_runs==1: + logger.info("PERFORMING 1 MORPHO PSEUDO-EXPERIMENT") + else: + logger.info("PERFORMING {} MORPHO PSEUDO-EXPERIMENTS".format(n_runs)) + + + if parallelize == False: + root_filenames = [] + for i in range(n_runs): + logger.info("----------MORPHO RUN #{}----------".format(i)) + temp_root_name = "./results/"+str(i)+root_basename + root_filenames.append(temp_root_name) + PerformFakeExperiment(temp_root_name) + else: + from multiprocessing import Pool + root_filenames = ["./results/"+str(i)+root_basename for i in range(n_runs)] + with Pool(n_processes) as p: + p.map(PerformFakeExperiment, root_filenames) + + #coverages = CalibrateResults(root_filenames, vars_to_calibrate) + coverages = None + return coverages + + +if __name__ == '__main__': + const_dict = { + # physical + 'Q':[QT2(),0.07], 'Q_std_analysis':75, 'mass':[1.1532, 0.4291], 'mass_init':0.2, + # instrumental + 'sigma': [15.524869238312053, 2.1583740056146], #[14.542692695653235, 1.3080022178297848], #1.301788807656317 + 'S':[3594], + 'err_from_B_generation':0, 'err_from_B_analysis':0.001, + 'B_1kev':[-3.057997933394048, 1.9966097834821164], 'B_1kev_init':0.0469817, 'B_init':0.06, + 'minf':1353.125e+06 - 40e+06 + 24.5e+09, 'f_nbins':65, + # complex lineshape + 'survival_prob_mean': 0.672621806411212, #0.688, #0.736, + 'survival_prob_err': 0.10760576977058844, #0.0368782, #0.00850261, + 'survival_prob': [12.118849968055722, 5.898481394892654], #[107.90268235294104, 48.93261176470582], #[2042.1165325779257, 926.0761019830128] + 'Bfield': [0.957809551203932, 8.498072412705302e-6], #[0.957805552, 1.3926736876423273e-6], #[0.957805552, 7.620531477410062e-7] + 'Nscatters_generation':20, 'Nscatters_analysis':16, + 'H2_scatter_prop_tritium':1., + # Stan fit + 'chain':1, 'warmup':2000, 'iter':3000, + 'KEscale':16323, 'adapt_delta':0.8 + } + + interest_vars = ['Q', 'mass','survival_prob', 'Bfield', 'sigma', 'S', 'B_1kev'] + + parser.add_argument("root_filename", type=str) + args = parser.parse_args() + + #CalibrateResults([args.root_filename], interest_vars, [0.16, 0.84]) + FakeExperimentEnsemble(3, args.root_filename, vars_to_calibrate=interest_vars) + From a582ef746756f9b7227dd813d82781a3c69e1d39 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Sun, 27 Sep 2020 13:31:16 -0400 Subject: [PATCH 015/199] use termite script for testing --- test_analysis/fake_data_stan_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_analysis/fake_data_stan_analysis.py b/test_analysis/fake_data_stan_analysis.py index 0ee91e6b..354aee0d 100644 --- a/test_analysis/fake_data_stan_analysis.py +++ b/test_analysis/fake_data_stan_analysis.py @@ -210,7 +210,7 @@ def SaveUnbinnedData(tritium_data, root_file="./results/tritium_analysis.root"): -def StanTritiumAnalysis(tritium_data, fit_parameters=None, root_file='./results/tritium_analysis.root', stan_files_location='../../morpho_models/', model_code='tritium_model/models/tritium_phase_II_analyzer_binned.stan', scattering_params_R='simplified_scattering_params.R'): +def StanTritiumAnalysis(tritium_data, fit_parameters=None, root_file='./results/tritium_analysis.root', stan_files_location='/host/morpho_models/', model_code='tritium_model/models/tritium_phase_II_analyzer_binned.stan', scattering_params_R='/host/simplified_scattering_params.R'): """ Analyzes frequency or kinetic energy data using a Stan model. Saves and plots posteriors. From e04a4f38fb0f69ef4c7c4ae85c61cfa5c35c5a68 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 28 Sep 2020 19:15:15 -0400 Subject: [PATCH 016/199] update the files --- mermithid/misc/ComplexLineShapeUtilities.py | 2 +- .../misc/MultiGasComplexLineShape.py | 205 ++++++++++++++++-- test_analysis/Complex_line_shape_fitter.py | 12 +- test_analysis/fake_data_stan_analysis.py | 4 +- 4 files changed, 190 insertions(+), 33 deletions(-) diff --git a/mermithid/misc/ComplexLineShapeUtilities.py b/mermithid/misc/ComplexLineShapeUtilities.py index 2a0be53d..ba60a7af 100644 --- a/mermithid/misc/ComplexLineShapeUtilities.py +++ b/mermithid/misc/ComplexLineShapeUtilities.py @@ -70,7 +70,7 @@ def get_eloss_spec(e_loss, oscillator_strength, kr_17keV_line): #energies in eV e_rydberg = 13.605693009 #rydberg energy (eV) a0 = 5.291772e-11 #bohr radius argument_of_log = np.where(e_loss > 0, 4. * kinetic_en * e_rydberg / (e_loss**2.) , 1e-5) - return np.where(e_loss>0 , 1./(e_loss) * oscillator_strength*np.log(argument_of_log), 0) + return np.where(e_loss>0 , 1./(e_loss) * oscillator_strength* np.log(argument_of_log), 0) # Takes only the nonzero bins of a histogram def get_only_nonzero_bins(bins,hist): diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 2db86a03..dae741a1 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -55,10 +55,7 @@ def InternalConfigure(self, params): self.fixed_scatter_proportion = reader.read_param(params, 'fixed_scatter_proportion', True) if self.fixed_scatter_proportion == True: self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) - self.fit_ftc = reader.read_param(params, 'fit_ftc', True) - if self.fit_ftc == True: - self.A_array = reader.read_param(params, 'A_array', []) - self.sigma_array = reader.read_param(params, 'sigma_array', []) + self.fit_ftc = reader.read_param(params, 'fit_ftc', True) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', False) # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown @@ -100,7 +97,10 @@ def InternalRun(self): else: self.results = self.fit_data_1(freq_bins, data_hist_freq) else: - self.results = self.fit_data(freq_bins, data_hist_freq) + if self.fit_ftc == True: + self.results = self.fit_data_ftc_1(freq_bins, data_hist_freq) + else: + self.results = self.fit_data(freq_bins, data_hist_freq) return True @@ -282,7 +282,7 @@ def convolve_composite_gaussian(self, func_to_convolve, A_array, sigma_array): return ans_normed def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): - ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt+'ins_resolution_all.txt') + ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt+'ins_resolution_all4.txt') x_data = ins_resolution_data.T[0] y_data = ins_resolution_data.T[1] y_err_data = ins_resolution_data.T[2] @@ -316,7 +316,10 @@ def chi2_Poisson(self, bin_centers, data_hist_freq, params): nonzero_bins_index = np.where(data_hist_freq != 0) zero_bins_index = np.where(data_hist_freq == 0) # expectation - fit_Hz = self.spectrum_func_ftc(bin_centers, *params) + if self.fit_ftc: + fit_Hz = self.spectrum_func_ftc(bin_centers, *params) + else: + fit_Hz = self.spectrum_func_1(bin_centers, *params) chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() return chi2 @@ -568,8 +571,7 @@ def spectrum_func_1(self, bins_Hz, *p0): nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] full_spectrum = self.make_spectrum_1(FWHM_G_eV, prob_parameter,) - full_spectrum_rev = ComplexLineShapeUtilities.flip_array(full_spectrum) - f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum_rev) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum) f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) return f @@ -580,7 +582,7 @@ def fit_data_1(self, freq_bins, data_hist_freq): bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) bins_Hz_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_Hz, data_hist_freq) # Initial guesses for curve_fit - FWHM_guess = 5 + FWHM_eV_guess = 5 B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) amplitude_guess = np.sum(data_hist_freq)/2 prob_parameter_guess = 0.5 @@ -594,11 +596,21 @@ def fit_data_1(self, freq_bins, data_hist_freq): prob_parameter_min = 1e-5 prob_parameter_max = 1 - p0_guess = [B_field_guess, FWHM_guess, amplitude_guess, prob_parameter_guess] - p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min], - [B_field_max, FWHM_eV_max, amplitude_max, prob_parameter_max]) + # p0_guess = [B_field_guess, FWHM_guess, amplitude_guess, prob_parameter_guess] + # p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min], + # [B_field_max, FWHM_eV_max, amplitude_max, prob_parameter_max]) # Actually do the fitting - params , cov = curve_fit(self.spectrum_func_1, bins_Hz_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) + # params , cov = curve_fit(self.spectrum_func_1, bins_Hz_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) + p0_guess = [B_field_guess, FWHM_eV_guess, amplitude_guess, prob_parameter_guess] + p0_bounds = [(B_field_min,B_field_max), (FWHM_eV_min, FWHM_eV_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max)] + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi2_Poisson(bins_Hz, data_hist_freq, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() # Name each of the resulting parameters and errors ################### Generalize to N Gases ########################### B_field_fit = params[0] @@ -607,7 +619,7 @@ def fit_data_1(self, freq_bins, data_hist_freq): prob_parameter_fit = params[3] total_counts_fit = amplitude_fit - perr = np.sqrt(np.diag(cov)) + perr = m_binned.np_errors() B_field_fit_err = perr[0] FWHM_eV_fit_err = perr[1] amplitude_fit_err = perr[2] @@ -625,8 +637,7 @@ def fit_data_1(self, freq_bins, data_hist_freq): data_Hz_nonzero = data_hist_freq[nonzero_bins_index] fit_Hz_zero = fit_Hz[zero_bins_index] data_Hz_zero = data_hist_freq[zero_bins_index] - chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) - reduced_chi2 = chi2/(len(data_hist_freq)-4) + reduced_chi2 = self.reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = 4) elapsed = time.time() - t output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -643,7 +654,6 @@ def fit_data_1(self, freq_bins, data_hist_freq): output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' dictionary_of_fit_results = { 'output_string': output_string, - 'cov': cov, 'bins_keV': bins_keV, 'fit_keV': fit_keV, 'bins_Hz': bins_Hz, @@ -660,7 +670,7 @@ def fit_data_1(self, freq_bins, data_hist_freq): 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results - + def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): gases = self.gases current_path = self.path_to_scatter_spectra_file @@ -739,8 +749,6 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess] p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max)] - logger.info(p0_guess) - logger.info(p0_bounds) # Actually do the fitting m_binned = Minuit.from_array_func(lambda p: self.chi2_Poisson(bins_Hz, data_hist_freq, p), start = p0_guess, @@ -767,7 +775,7 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) - reduced_chi2 = self.reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = 4) + reduced_chi2 = self.reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = 3) elapsed = time.time() - t output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -796,4 +804,155 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): 'data_hist_freq': data_hist_freq, 'reduced_chi2': reduced_chi2 } - return dictionary_of_fit_results \ No newline at end of file + return dictionary_of_fit_results + + + def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak='shake'): + gases = self.gases + current_path = self.path_to_scatter_spectra_file + # check_existence_of_scatter_files() + #filenames = list_files('scatter_spectra_files') + p = np.zeros(len(gases)) + p[0:-1] = scatter_proportion + p[-1] = 1 - sum(scatter_proportion) + scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') + scatter_spectra = np.load( + scatter_spectra_file_path, allow_pickle = True + ) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += current_working_spectrum + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(len(self.gases))): + coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M + current_full_spectrum += coefficient*current_working_spectrum + return current_full_spectrum + + def spectrum_func_ftc_2(self, bins_Hz, *p0): + B_field = p0[0] + amplitude = p0[1] + prob_parameter = p0[2] + N = len(self.gases) + scatter_proportion = p0[3:2+N] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + en_array_rev = ComplexLineShapeUtilities.flip_array(-1*en_loss_array) + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_ftc_1(prob_parameter, scatter_proportion) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum) + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + return f + + def fit_data_ftc_2(self, freq_bins, data_hist_freq): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + bins_Hz_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_Hz, data_hist_freq) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + N = len(self.gases) + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess] + (N-1)*[scatter_proportion_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max)] + (N-1)*[(scatter_proportion_min, scatter_proportion_max)] + logger.info(p0_guess) + logger.info(p0_bounds) + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi2_Poisson(bins_Hz, data_hist_freq, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + # Name each of the resulting parameters and errors + ################### Generalize to N Gases ########################### + B_field_fit = params[0] + amplitude_fit = params[1] + prob_parameter_fit = params[2] + scatter_proportion_fit = list(params[3:2+N])+[1- sum(params[3:2+N])] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + prob_parameter_fit_err = perr[2] + scatter_proportion_fit_err = list(perr[3:2+N])+[np.sqrt(sum(perr[3:2+N]**2))] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_ftc_1(bins_Hz, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + + reduced_chi2 = self.reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = 4) + elapsed = time.time() - t + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)+' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' + output_string += '-----------------\n' + for i in range(len(self.gases)): + output_string += '{} Scatter proportion \n= '.format(self.gases[i]) + "{:.8e}".format(scatter_proportion_fit[i])\ + +' +/- ' + "{:.2e}".format(scatter_proportion_fit_err[i])+'\n' + output_string += '-----------------\n' + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'prob_parameter_fit': prob_parameter_fit, + 'prob_parameter_fit_err': prob_parameter_fit_err, + 'scatter_proportion_fit': scatter_proportion_fit, + 'scatter_proportion_fit_err': scatter_proportion_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 8d5db099..3edaa407 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -28,15 +28,13 @@ def test_complex_lineshape(self): "variables": ['StartTimeInAcq','StartFrequency'] } complexLineShape_config = { - 'bins_choice': np.linspace(0e6, 90e6, 1000), - 'gases': ["H2", "He"], + 'bins_choice': np.linspace(0e6, 100e6, 1000), + 'gases': ["H2", "Ar"], 'max_scatters': 20, 'fixed_scatter_proportion': True, + 'fit_ftc':True, # use gaussian instrumental resolution # When fix_scatter_proportion is True, set the scatter proportion for gas1 below 'gas_scatter_proportion': [0.93, 0.02], - 'fit_ftc': True, - 'A_array': [0.076, 0.341, 0.381, 0.203], - 'sigma_array': [5.01, 13.33, 15.40, 11.85], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -77,7 +75,7 @@ def test_complex_lineshape(self): plt.plot(results['bins_Hz']/1e9, results['fit_Hz'], label = results['output_string'], alpha = 0.7) plt.legend(loc = 'upper left', fontsize = 12) plt.xlabel('frequency GHz') - plot_title = 'fit ftc oct with {} gas scattering'.format(len(complexLineShape_config['gases'])) + plot_title = 'fit ftc march with {} gas scattering'.format(len(complexLineShape_config['gases'])) if complexLineShape_config['fixed_scatter_proportion'] == True: str_gas_scatter_proportion = '' for i in range(len(complexLineShape_config['gases'])): @@ -88,7 +86,7 @@ def test_complex_lineshape(self): plot_title += '\n with fixed scatter proportion \n {}'.format(str_gas_scatter_proportion) plt.title(plot_title) plt.tight_layout() - plt.savefig('/host/plots/fit_FTC_march_with_{}_gas_scattering.png'.format(len(complexLineShape_config['gases']))) + plt.savefig('/host/plots/fit_FTC_march_with_{}_gas_scattering_gaussian_ins_res.png'.format(len(complexLineShape_config['gases']))) if __name__ == '__main__': diff --git a/test_analysis/fake_data_stan_analysis.py b/test_analysis/fake_data_stan_analysis.py index 00ebdce0..0ee91e6b 100644 --- a/test_analysis/fake_data_stan_analysis.py +++ b/test_analysis/fake_data_stan_analysis.py @@ -112,7 +112,7 @@ def GenerateFakeData(inputs_dict): """ specGen_config = { "apply_efficiency": True, - "efficiency_path": "../phase2_detection_efficiency_curve/combined_energy_corrected_count_rates/combined_energy_corrected_eff_at_quad_trap_frequencies.json", + "efficiency_path": "../tests/combined_energy_corrected_eff_at_quad_trap_frequencies.json", "detailed_or_simplified_lineshape": "detailed", "return_frequency": True, "Q": inputs_dict["Q"], @@ -154,7 +154,7 @@ def BinAndSaveData(tritium_data, nbins, root_file="./results/tritium_analysis.ro "energy_or_frequency": 'frequency', "variables": "F", "title": "corrected_spectrum", - "efficiency_filepath": "../phase2_detection_efficiency_curve/combined_energy_corrected_count_rates/combined_energy_corrected_eff_at_quad_trap_frequencies.json", + "efficiency_filepath": "../tests/combined_energy_corrected_eff_at_quad_trap_frequencies.json", 'bins': np.linspace(tritium_data['minf'], tritium_data['maxf'], nbins), 'fss_bins': False # If fss_bins is True, bins is ignored and overridden } From be09e5aa6951ebacfbd0a48295f468bffb14f96d Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 28 Sep 2020 19:30:34 -0400 Subject: [PATCH 017/199] delete configuration for magnetic field --- test_analysis/Complex_line_shape_fitter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 3edaa407..0f89c016 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -39,7 +39,6 @@ def test_complex_lineshape(self): # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, 'RF_ROI_MIN': 25850000000.0, - 'B_field': 0.957810722501, # shake_spectrum_parameters.json and oscillator strength data can be found at https://github.com/project8/scripts/tree/master/yuhao/line_shape_fitting/data 'shake_spectrum_parameters_json_path': '../mermithid/misc/shake_spectrum_parameters.json', 'path_to_osc_strengths_files': '/host/', From 5825a964b850e5a81b7ed62ae3eb365c2fbb035b Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 30 Sep 2020 21:09:53 -0400 Subject: [PATCH 018/199] make path_to_ins_resolution_data_txt configurable in fake_data_stan_analysis.py configurable --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 2 ++ mermithid/processors/misc/KrComplexLineShape.py | 4 ++-- test_analysis/fake_data_stan_analysis.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index f391579d..09c1a6e7 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -103,6 +103,7 @@ def InternalConfigure(self, params): #paths self.simplified_scattering_path = reader.read_param(params, 'simplified_scattering_path', '/host/input_data/simplified_scattering_params.txt') self.detailed_scatter_spectra_path = reader.read_param(params, 'path_to_detailed_scatter_spectra_dir', '/host') + self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all4.txt') self.efficiency_path = reader.read_param(params, 'efficiency_path', '') #options @@ -157,6 +158,7 @@ def InternalConfigure(self, params): 'B_field': self.B_field, 'base_shape': 'dirac', 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path + 'path_to_ins_resolution_data_txt': self.path_to_ins_resolution_data_txt } logger.info('Setting up complex lineshape object') self.complexLineShape = KrComplexLineShape("complexLineShape") diff --git a/mermithid/processors/misc/KrComplexLineShape.py b/mermithid/processors/misc/KrComplexLineShape.py index eabc5383..43ba0104 100644 --- a/mermithid/processors/misc/KrComplexLineShape.py +++ b/mermithid/processors/misc/KrComplexLineShape.py @@ -63,7 +63,7 @@ def InternalConfigure(self, params): self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') self.base_shape = reader.read_param(params, 'base_shape', 'shake') self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') - self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/') + self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all4.txt') if self.base_shape=='shake' and not os.path.exists(self.shake_spectrum_parameters_json_path): raise IOError('Shake spectrum path does not exist') @@ -253,7 +253,7 @@ def convolve_gaussian(self, func_to_convolve,gauss_FWHM_eV): return ans_normed def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): - ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt+'ins_resolution_all4.txt') + ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt) x_data = ins_resolution_data.T[0] y_data = ins_resolution_data.T[1] y_err_data = ins_resolution_data.T[2] diff --git a/test_analysis/fake_data_stan_analysis.py b/test_analysis/fake_data_stan_analysis.py index 354aee0d..a29489c5 100644 --- a/test_analysis/fake_data_stan_analysis.py +++ b/test_analysis/fake_data_stan_analysis.py @@ -113,6 +113,7 @@ def GenerateFakeData(inputs_dict): specGen_config = { "apply_efficiency": True, "efficiency_path": "../tests/combined_energy_corrected_eff_at_quad_trap_frequencies.json", + 'path_to_ins_resolution_data_txt': '/host/ins_resolution_all4.txt' "detailed_or_simplified_lineshape": "detailed", "return_frequency": True, "Q": inputs_dict["Q"], From 352c67247495db776a21d76c479e376faa432031 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Sun, 4 Oct 2020 20:33:48 -0400 Subject: [PATCH 019/199] test resolution function configurable --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 09c1a6e7..ef075567 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -157,7 +157,7 @@ def InternalConfigure(self, params): 'num_points_in_std_array': 10000, 'B_field': self.B_field, 'base_shape': 'dirac', - 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path + 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path, 'path_to_ins_resolution_data_txt': self.path_to_ins_resolution_data_txt } logger.info('Setting up complex lineshape object') From d5ddea3300015d1f55c3d9cd505318f66b8015d9 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Fri, 9 Oct 2020 15:17:04 -0400 Subject: [PATCH 020/199] Loading/using simulated resolution file --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index f391579d..8c729609 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -56,6 +56,7 @@ def InternalConfigure(self, params): - simplified_scattering_path: path to simplified lineshape parameters - path_to_detailed_scatter_spectra_dir: path to oscillator and or scatter_spectra_file - efficiency_path: path to efficiency vs. frequency (and uncertainties) + - ins_res_path: path to file with simulated instrumental resolution data - use_lineshape (boolean): determines whether tritium spectrum is smeared by lineshape. If False, it will only be smeared with a Gaussian - detailed_or_simplified_lineshape: If use lineshape, this string determines which lineshape model is used. @@ -104,6 +105,7 @@ def InternalConfigure(self, params): self.simplified_scattering_path = reader.read_param(params, 'simplified_scattering_path', '/host/input_data/simplified_scattering_params.txt') self.detailed_scatter_spectra_path = reader.read_param(params, 'path_to_detailed_scatter_spectra_dir', '/host') self.efficiency_path = reader.read_param(params, 'efficiency_path', '') + self.ins_resolution_data_path = reader.read_param(params, 'ins_res_path', '') #options self.use_lineshape = reader.read_param(params, 'use_lineshape', True) @@ -156,7 +158,8 @@ def InternalConfigure(self, params): 'num_points_in_std_array': 10000, 'B_field': self.B_field, 'base_shape': 'dirac', - 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path + 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path, + 'path_to_ins_resolution_data_txt': self.ins_resolution_data_path } logger.info('Setting up complex lineshape object') self.complexLineShape = KrComplexLineShape("complexLineShape") From 186ee0f1e2dc347c27b2aa57b942537c12c2615c Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Fri, 9 Oct 2020 15:51:21 -0400 Subject: [PATCH 021/199] Sample simulated resolution counts rates to account for errors --- mermithid/processors/misc/KrComplexLineShape.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/misc/KrComplexLineShape.py b/mermithid/processors/misc/KrComplexLineShape.py index 43ba0104..980be3c8 100644 --- a/mermithid/processors/misc/KrComplexLineShape.py +++ b/mermithid/processors/misc/KrComplexLineShape.py @@ -63,7 +63,7 @@ def InternalConfigure(self, params): self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') self.base_shape = reader.read_param(params, 'base_shape', 'shake') self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') - self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all4.txt') + self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all.txt') if self.base_shape=='shake' and not os.path.exists(self.shake_spectrum_parameters_json_path): raise IOError('Shake spectrum path does not exist') @@ -260,7 +260,9 @@ def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): return x_data, y_data, y_err_data def convolve_ins_resolution(self, working_spectrum): - x_data, y_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + x_data, y_mean_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + y_data = np.random.normal(y_mean_data, y_err_data) + y_data[y_data<0] = 0 f = interpolate.interp1d(x_data, y_data) x_array = self.std_eV_array() y_array = np.zeros(len(x_array)) From 10c08f3e32b6fa63ebe486fa9861eb3227642b8f Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Sun, 18 Oct 2020 17:59:20 -0400 Subject: [PATCH 022/199] add MultiGAsComplexLineShape.py --- .../misc/MultiGasComplexLineShape.py | 965 ++++++++++++++++++ 1 file changed, 965 insertions(+) create mode 100644 mermithid/processors/misc/MultiGasComplexLineShape.py diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py new file mode 100644 index 00000000..ea7a4af0 --- /dev/null +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -0,0 +1,965 @@ +''' +Fits data to complex lineshape model. +Author: E. Machado, Y.-H. Sun, E. Novitski +Date: 4/8/20 + +This processor takes in frequency data in binned histogram and fit the histogram with two gas scattering complex line shape model. + +Configurable parameters: + +There are two options available for fitting: fix_scatter_proportion = True and False. +gases: array for names of the two gases involved in the scattering process. +max_scatter: max number of scatterings for only single gas scatterings. +max_comprehansive_scatter: max number of scatterings for all cross scatterings. +scatter_proportion: when fix_scatter_proportion is set as true, gives the fixed scatter proportion. +num_points_in_std_array: number of points for std_array defining how finely the scatter calculations are. +RF_ROI_MIN: can be found from meta data. +B_field: can be put in hand or found by position of the peak of the frequency histogram. +shake_spectrum_parameters_json_path: path to json file storing shake spectrum parameters. +path_to_osc_strength_files: path to oscillator strength files. +''' + +from __future__ import absolute_import + +import numpy as np +from scipy.optimize import curve_fit +from scipy.special import comb +from scipy import integrate , signal, interpolate +from itertools import product +from math import factorial +from iminuit import Minuit +import os +import time +import sys +from morpho.utilities import morphologging, reader +from morpho.processors import BaseProcessor +from mermithid.misc import Constants, ComplexLineShapeUtilities, ConversionFunctions + +logger = morphologging.getLogger(__name__) + + + +__all__ = [] +__all__.append(__name__) + +class MultiGasComplexLineShape(BaseProcessor): + + def InternalConfigure(self, params): + ''' + Configure + ''' + # Read other parameters + self.bins_choice = reader.read_param(params, 'bins_choice', []) + self.gases = reader.read_param(params, 'gases', ["H2", "Kr", "He", "Ar"]) + self.max_scatters = reader.read_param(params, 'max_scatters', 20) + self.fixed_scatter_proportion = reader.read_param(params, 'fixed_scatter_proportion', True) + if self.fixed_scatter_proportion == True: + self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) + self.fit_ftc = reader.read_param(params, 'fit_ftc', True) + self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', False) + # This is an important parameter which determines how finely resolved + # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown + self.num_points_in_std_array = reader.read_param(params, 'num_points_in_std_array', 10000) + self.RF_ROI_MIN = reader.read_param(params, 'RF_ROI_MIN', 25850000000.0) + self.B_field = reader.read_param(params, 'B_field', 0.957810722501) + self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') + self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') + self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') + self.path_to_missing_track_radiation_loss_data_numpy_file = '/host/' + self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all4.txt') + + if not os.path.exists(self.shake_spectrum_parameters_json_path): + raise IOError('Shake spectrum path does not exist') + if not os.path.exists(self.path_to_osc_strengths_files): + raise IOError('Path to osc strengths files does not exist') + return True + + def InternalRun(self): + + # Read shake parameters from JSON file + self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) + + # number_of_events = len(self.data['StartFrequency']) + # self.results = number_of_events + + a = self.data['StartFrequency'] + + # fit with shake spectrum + data_hist_freq, freq_bins= np.histogram(a,bins=self.bins_choice) + # histogram = data_hist_freq +# bins = freq_bins +# guess = np.where(np.array(histogram) == np.max(histogram))[0][0] +# kr17kev_in_hz = guess*(bins[1]-bins[0])+bins[0] + #self.B_field = B(17.8, kr17kev_in_hz + 0) + if self.fixed_scatter_proportion == True: + if self.fit_ftc == True: + self.results = self.fit_data_ftc(freq_bins, data_hist_freq) + else: + self.results = self.fit_data_1(freq_bins, data_hist_freq) + else: + if self.fit_ftc == True: + self.results = self.fit_data_ftc_2(freq_bins, data_hist_freq) + else: + self.results = self.fit_data(freq_bins, data_hist_freq) + + return True + + + # Establishes a standard energy loss array (SELA) from -1000 eV to 1000 eV + # with number of points equal to self.num_points_in_std_array. All convolutions + # will be carried out on this particular discretization + def std_eV_array(self): + emin = -1000 + emax = 1000 + array = np.linspace(emin,emax,self.num_points_in_std_array) + return array + + # A lorentzian line centered at 0 eV, with 2.83 eV width on the SELA + def std_lorenztian_17keV(self): + x_array = self.std_eV_array() + ans = lorentzian(x_array,0,kr_line_width) + return ans + + # A gaussian function + def gaussian(self, x_array, A, sigma, mu): + f = A*(1./(sigma*np.sqrt(2*np.pi)))*np.exp(-(((x_array-mu)/sigma)**2.)/2.) + return f + + # A gaussian centered at 0 eV with variable width, on the SELA + def std_gaussian(self, sigma): + x_array = self.std_eV_array() + ans = ComplexLineShapeUtilities.gaussian(x_array,1,sigma,0) + return ans + + def composite_gaussian(self, A_array, sigma_array): + x_array = self.std_eV_array() + ans = 0 + for A, sigma in zip(A_array, sigma_array): + ans += self.gaussian(x_array, A, sigma, 0) + return ans + + # normalizes a function, but depends on binning. + # Only to be used for functions evaluated on the SELA + def normalize(self, f): + x_arr = self.std_eV_array() + f_norm = integrate.simps(f,x=x_arr) + f_normed = f/f_norm + return f_normed + + # Function for energy loss from a single scatter of electrons by + # V.N. Aseev et al. 2000 + # This function does the work of combining fit_func1 and fit_func2 by + # finding the point where they intersect. + # Evaluated on the SELA + def single_scatter_f(self, gas_type): + energy_loss_array = self.std_eV_array() + f = 0 * energy_loss_array + + input_filename = self.path_to_osc_strengths_files + gas_type + "OscillatorStrength.txt" + energy_fOsc = ComplexLineShapeUtilities.read_oscillator_str_file(input_filename) + fData = interpolate.interp1d(energy_fOsc[0], energy_fOsc[1], kind='linear') + for i in range(len(energy_loss_array)): + if energy_loss_array[i] < energy_fOsc[0][0]: + f[i] = 0 + elif energy_loss_array[i] <= energy_fOsc[0][-1]: + f[i] = fData(energy_loss_array[i]) + else: + f[i] = ComplexLineShapeUtilities.aseev_func_tail(energy_loss_array[i], gas_type) + + f_e_loss = ComplexLineShapeUtilities.get_eloss_spec(energy_loss_array, f, Constants.kr_k_line_e()) + f_normed = self.normalize(f_e_loss) + return f_normed + + # Convolves a function with the single scatter function, on the SELA + def another_scatter(self, input_spectrum, gas_type): + single = self.single_scatter_f(gas_type) + f = signal.convolve(single,input_spectrum,mode='same') + f_normed = self.normalize(f) + return f_normed + + def radiation_loss_f(self): + radiation_loss_data_file_path = self.path_to_missing_track_radiation_loss_data_numpy_file + '/missing_track_radiation_loss.npy' + data_for_missing_track_radiation_loss = np.load(radiation_loss_data_file_path, allow_pickle = True) + x_data_for_histogram = data_for_missing_track_radiation_loss.item()['histogram_eV']['x_data'] + energy_loss_array = self.std_eV_array() + f_radiation_energy_loss = 0 * energy_loss_array + f_radiation_energy_loss_interp = data_for_missing_track_radiation_loss.item()['histogram_eV']['interp'] + for i in range(len(energy_loss_array)): + if energy_loss_array[i] >= x_data_for_histogram[0] and energy_loss_array[i] <= x_data_for_histogram[-1]: + f_radiation_energy_loss[i] = f_radiation_energy_loss_interp(energy_loss_array[i]) + else: + f_radiation_energy_loss[i] = 0 + return f_radiation_energy_loss + + # Convolves the scatter functions and saves + # the results to a .npy file. + def generate_scatter_convolution_file(self): + t = time.time() + scatter_spectra_single_gas = {} + for gas_type in self.gases: + f_radiation_loss = self.radiation_loss_f() + scatter_spectra_single_gas[gas_type] = {} + first_scatter = self.single_scatter_f(gas_type) + if self.use_radiation_loss == True: + first_scatter = self.normalize(signal.convolve(first_scatter, f_radiation_loss, mode = 'same')) + scatter_num_array = range(2, self.max_scatters+1) + current_scatter = first_scatter + scatter_spectra_single_gas[gas_type][str(1).zfill(2)] = current_scatter + # x = std_eV_array() # diagnostic + for i in scatter_num_array: + current_scatter = self.another_scatter(current_scatter, gas_type) + if self.use_radiation_loss == True: + current_scatter = self.normalize(signal.convolve(current_scatter, f_radiation_loss, mode = 'same')) + scatter_spectra_single_gas[gas_type][str(i).zfill(2)] = current_scatter + N = len(self.gases) + scatter_spectra = {} + for M in range(1, self.max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + mark_first_nonzero_component = 0 + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + if component == 0: + continue + else: + if mark_first_nonzero_component == 0: + current_full_scatter = scatter_spectra_single_gas[gas_type][str(component).zfill(2)] + mark_first_nonzero_component = 1 + else: + scatter_to_add = scatter_spectra_single_gas[gas_type][str(component).zfill(2)] + current_full_scatter = self.normalize(signal.convolve(current_full_scatter, scatter_to_add, mode='same')) + scatter_spectra[entry_str] = current_full_scatter + np.save(os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy'), scatter_spectra) + elapsed = time.time() - t + logger.info('Files generated in '+str(elapsed)+'s') + return + + # Checks for the existence of a directory called 'scatter_spectra_file' + # and checks that this directory contains the scatter spectra files. + # If not, this function calls generate_scatter_convolution_file. + # This function also checks to make sure that the scatter file have the correct + # number of entries and correct number of points in the SELA, and if not, it generates a fresh file. + # When the variable regenerate is set as True, it generates a fresh file + def check_existence_of_scatter_file(self, regenerate = True): + gases = self.gases + if regenerate == True: + logger.info('generate fresh scatter file') + self.generate_scatter_convolution_file() + else: + directory = os.listdir(self.path_to_scatter_spectra_file) + strippeddirs = [s.strip('\n') for s in directory] + if 'scatter_spectra.npy' not in strippeddirs: + self.generate_scatter_convolution_file() + test_file = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + test_dict = np.load(test_file, allow_pickle = True) + N = len(self.gases) + if len(test_dict.item()) != sum([comb(M + N -1, N -1) for M in range(1, self.max_scatters+1)]): + logger.info('Number of scatter combinations not matching, generating fresh files') + self.generate_scatter_convolution_file() + test_dict = np.load(test_file, allow_pickle = True) + gas_str = gases[0] + '01' + for gas in self.gases[1:]: + gas_str += gas + '00' + if gas_str not in list(test_dict.item().keys()): + print('Gas species not matching, generating fresh files') + generate_scatter_convolution_files() + return + + # Given a function evaluated on the SELA, convolves it with a gaussian + def convolve_gaussian(self, func_to_convolve,gauss_FWHM_eV): + sigma = ComplexLineShapeUtilities.gaussian_FWHM_to_sigma(gauss_FWHM_eV) + resolution_f = self.std_gaussian(sigma) + ans = signal.convolve(resolution_f,func_to_convolve,mode='same') + ans_normed = self.normalize(ans) + return ans_normed + + def convolve_composite_gaussian(self, func_to_convolve, A_array, sigma_array): + resolution_f = self.composite_gaussian(A_array, sigma_array) + ans = signal.convolve(resolution_f, func_to_convolve, mode='same') + ans_normed = self.normalize(ans) + return ans_normed + + def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): + ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt) + x_data = ins_resolution_data.T[0] + y_data = ins_resolution_data.T[1] + y_err_data = ins_resolution_data.T[2] + return x_data, y_data, y_err_data + + def convolve_ins_resolution(self, working_spectrum): + x_data, y_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + f = interpolate.interp1d(x_data, y_data) + x_array = self.std_eV_array() + y_array = np.zeros(len(x_array)) + index_within_range_of_xdata = np.where((x_array >= x_data[0]) & (x_array <= x_data[-1])) + y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]) + convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') + normalized_convolved_spectrum = self.normalize(convolved_spectrum) + return normalized_convolved_spectrum + + def least_square(self, bin_centers, hist, params): + # expectation + expectation = self.spectrum_func_ftc(bin_centers, *params) + + high_count_index = np.where(hist>0) + #low_count_index = np.where((hist>0) & (hist<=50)) + zero_count_index = np.where(hist==0) + + lsq = ((hist[high_count_index]- expectation[high_count_index])**2/hist[high_count_index]).sum() + #lsq += ((hist[low_count_index]- expectation[low_count_index])**2/hist[low_count_index]).sum() + lsq += ((hist[zero_count_index]- expectation[zero_count_index])**2).sum() + return lsq + + def chi2_Poisson(self, bin_centers, data_hist_freq, params): + nonzero_bins_index = np.where(data_hist_freq != 0) + zero_bins_index = np.where(data_hist_freq == 0) + # expectation + if self.fixed_scatter_proportion: + if self.fit_ftc: + fit_Hz = self.spectrum_func_ftc(bin_centers, *params) + else: + fit_Hz = self.spectrum_func_1(bin_centers, *params) + else: + if self.fit_ftc: + fit_Hz = self.spectrum_func_ftc_2(bin_centers, *params) + else: + fit_Hz = self.spectrum_func(bin_centers, *params) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + return chi2 + + def reduced_chi2_Pearson_Neyman_composite(self, data_hist_freq, fit_Hz, number_of_parameters): + nonzero_bins_index = np.where(data_hist_freq != 0)[0] + zero_bins_index = np.where(data_hist_freq == 0)[0] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_zero = fit_Hz[zero_bins_index] + data_Hz_zero = data_hist_freq[zero_bins_index] + chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) + reduced_chi2 = chi2/(len(data_hist_freq) - number_of_parameters) + return reduced_chi2 + + # following the expression in the paper Steve BAKER and Robert D. COUSINS, (1984) CLARIFICATION OF THE USE OF CHI-SQUARE AND LIKELIHOOD FUNCTIONS IN FITS TO HISTOGRAMS + def reduced_chi2_Poisson(self, data_hist_freq, fit_Hz, number_of_parameters): + nonzero_bins_index = np.where(data_hist_freq != 0) + zero_bins_index = np.where(data_hist_freq == 0) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + reduced_chi2 = chi2/(len(data_hist_freq) - number_of_parameters) + return reduced_chi2 + + def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitted_peak='shake'): + gases = self.gases + max_scatters = self.max_scatters + current_path = self.path_to_scatter_spectra_file + # check_existence_of_scatter_files() + #filenames = list_files('scatter_spectra_files') + p = np.zeros(len(gases)) + p[0:-1] = scatter_proportion + p[-1] = 1 - sum(scatter_proportion) + scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') + scatter_spectra = np.load( + scatter_spectra_file_path, allow_pickle = True + ) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_gaussian(current_working_spectrum, gauss_FWHM_eV) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += current_working_spectrum + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(len(self.gases))): + coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M + current_full_spectrum += coefficient*current_working_spectrum + return current_full_spectrum + + # Produces a spectrum in real energy that can now be evaluated off of the SELA. + #def spectrum_func(x_keV,FWHM_G_eV,line_pos_keV,scatter_prob,amplitude): + def spectrum_func(self, bins_Hz, *p0): + B_field = p0[0] + FWHM_G_eV = p0[1] + amplitude = p0[2] + prob_parameter = p0[3] + N = len(self.gases) + scatter_proportion = p0[4:3+N] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + en_array_rev = ComplexLineShapeUtilities.flip_array(-1*en_loss_array) + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum(FWHM_G_eV, prob_parameter, scatter_proportion) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum) + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + return f + + # Call this function to fit a histogram of start frequencies with the model. + # Note that the data_hist_freq should be the StartFrequencies as given by katydid, + # which will be from ~0 MHZ to ~100 MHz. You must also pass this function the + # self.RF_ROI_MIN value from the metadata file of your data. + # You must also supply a guess for the self.B_field present for the run; + # 0.959 T is usually sufficient. + + def fit_data(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + bins_Hz_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_Hz, data_hist_freq) + # Initial guesses for curve_fit + FWHM_guess = 5 + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + FWHM_eV_min = 1e-5 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) - ConversionFunctions.Energy(bins_Hz[-1], B_field_guess) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + N = len(self.gases) + p0_guess = [B_field_guess, FWHM_guess, amplitude_guess, prob_parameter_guess] + [scatter_proportion_guess]*(N-1) + p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min] + [scatter_proportion_min]*(N-1), + [B_field_max, FWHM_eV_max, amplitude_max, prob_parameter_max] + [scatter_proportion_max]*(N-1) ) + # Actually do the fitting + params , cov = curve_fit(self.spectrum_func, bins_Hz_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) + # Name each of the resulting parameters and errors + ################### Generalize to N Gases ########################### + B_field_fit = params[0] + FWHM_eV_fit = params[1] + amplitude_fit = params[2] + prob_parameter_fit = params[3] + #starting at index 4, grabs every other entry. (which is how scattering probs are filled in for N gases) + scatter_proportion_fit = list(params[4:3+N])+[1- sum(params[4:3+N])] + total_counts_fit = amplitude_fit + + perr = np.sqrt(np.diag(cov)) + B_field_fit_err = perr[0] + FWHM_eV_fit_err = perr[1] + amplitude_fit_err = perr[2] + prob_parameter_fit_err = perr[3] + scatter_proportion_fit_err = list(perr[4:3+N])+[np.sqrt(sum(perr[4:3+N]**2))] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func(bins_Hz, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + + nonzero_bins_index = np.where(data_hist_freq != 0)[0] + zero_bins_index = np.where(data_hist_freq == 0)[0] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_zero = fit_Hz[zero_bins_index] + data_Hz_zero = data_hist_freq[zero_bins_index] + chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) + reduced_chi2 = chi2/(len(data_hist_freq)-4-len(self.gases)+1) + elapsed = time.time() - t + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Gaussian FWHM = '+str(round(FWHM_eV_fit,2))+' +/- '+str(round(FWHM_eV_fit_err,2))+' eV\n' + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)\ + +' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' + output_string += '-----------------\n' + for i in range(len(self.gases)): + output_string += '{} Scatter proportion \n= '.format(self.gases[i]) + "{:.8e}".format(scatter_proportion_fit[i])\ + +' +/- ' + "{:.2e}".format(scatter_proportion_fit_err[i])+'\n' + output_string += '-----------------\n' + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'cov': cov, + 'bins_keV': bins_keV, + 'fit': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'FWHM_eV_fit': FWHM_eV_fit, + 'FWHM_eV_fit_err': FWHM_eV_fit_err, + 'prob_parameter_fit': prob_parameter_fit, + 'prob_parameter_fit_err': prob_parameter_fit_err, + 'scatter_proportion_fit': scatter_proportion_fit, + 'scatter_proportion_fit_err': scatter_proportion_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results + + def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak='shake'): + gases = self.gases + current_path = self.path_to_scatter_spectra_file + # check_existence_of_scatter_files() + #filenames = list_files('scatter_spectra_files') + p = np.zeros(len(gases)) + p = self.scatter_proportion + scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') + scatter_spectra = np.load( + scatter_spectra_file_path, allow_pickle = True + ) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_gaussian(current_working_spectrum, gauss_FWHM_eV) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += current_working_spectrum + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(len(self.gases))): + coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M + current_full_spectrum += coefficient*current_working_spectrum + return current_full_spectrum + + def spectrum_func_1(self, bins_Hz, *p0): + B_field = p0[0] + FWHM_G_eV = p0[1] + amplitude = p0[2] + prob_parameter = p0[3] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + en_array_rev = ComplexLineShapeUtilities.flip_array(-1*en_loss_array) + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_1(FWHM_G_eV, prob_parameter,) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum) + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + return f + + def fit_data_1(self, freq_bins, data_hist_freq): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + bins_Hz_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_Hz, data_hist_freq) + # Initial guesses for curve_fit + FWHM_eV_guess = 5 + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + prob_parameter_guess = 0.5 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + FWHM_eV_min = 1e-5 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) - ConversionFunctions.Energy(bins_Hz[-1], B_field_guess) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + + # p0_guess = [B_field_guess, FWHM_guess, amplitude_guess, prob_parameter_guess] + # p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min], + # [B_field_max, FWHM_eV_max, amplitude_max, prob_parameter_max]) + # Actually do the fitting + # params , cov = curve_fit(self.spectrum_func_1, bins_Hz_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) + p0_guess = [B_field_guess, FWHM_eV_guess, amplitude_guess, prob_parameter_guess] + p0_bounds = [(B_field_min,B_field_max), (FWHM_eV_min, FWHM_eV_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max)] + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi2_Poisson(bins_Hz, data_hist_freq, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + # Name each of the resulting parameters and errors + ################### Generalize to N Gases ########################### + B_field_fit = params[0] + FWHM_eV_fit = params[1] + amplitude_fit = params[2] + prob_parameter_fit = params[3] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + FWHM_eV_fit_err = perr[1] + amplitude_fit_err = perr[2] + prob_parameter_fit_err = perr[3] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_1(bins_Hz, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + + nonzero_bins_index = np.where(data_hist_freq != 0)[0] + zero_bins_index = np.where(data_hist_freq == 0)[0] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_zero = fit_Hz[zero_bins_index] + data_Hz_zero = data_hist_freq[zero_bins_index] + reduced_chi2 = self.reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = 4) + elapsed = time.time() - t + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Gaussian FWHM = '+str(round(FWHM_eV_fit,2))+' +/- '+str(round(FWHM_eV_fit_err,2))+' eV\n' + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)\ + +' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' + output_string += '-----------------\n' + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'FWHM_eV_fit': FWHM_eV_fit, + 'FWHM_eV_fit_err': FWHM_eV_fit_err, + 'prob_parameter_fit': prob_parameter_fit, + 'prob_parameter_fit_err': prob_parameter_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results + + def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): + gases = self.gases + current_path = self.path_to_scatter_spectra_file + # check_existence_of_scatter_files() + #filenames = list_files('scatter_spectra_files') + p = np.zeros(len(gases)) + p = self.scatter_proportion + scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') + scatter_spectra = np.load( + scatter_spectra_file_path, allow_pickle = True + ) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += current_working_spectrum + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(len(self.gases))): + coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M + current_full_spectrum += coefficient*current_working_spectrum + return current_full_spectrum + + def spectrum_func_ftc(self, bins_Hz, *p0): + B_field = p0[0] + amplitude = p0[1] + prob_parameter = p0[2] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + en_array_rev = ComplexLineShapeUtilities.flip_array(-1*en_loss_array) + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_ftc(prob_parameter) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum) + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + return f + + def fit_data_ftc(self, freq_bins, data_hist_freq): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + bins_Hz_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_Hz, data_hist_freq) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + prob_parameter_guess = 0.5 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max)] + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi2_Poisson(bins_Hz, data_hist_freq, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + # Name each of the resulting parameters and errors + ################### Generalize to N Gases ########################### + B_field_fit = params[0] + amplitude_fit = params[1] + prob_parameter_fit = params[2] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + prob_parameter_fit_err = perr[2] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_ftc(bins_Hz, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + + reduced_chi2 = self.reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = 3) + elapsed = time.time() - t + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)\ + +' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' + output_string += '-----------------\n' + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'prob_parameter_fit': prob_parameter_fit, + 'prob_parameter_fit_err': prob_parameter_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results + + + def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak='shake'): + gases = self.gases + current_path = self.path_to_scatter_spectra_file + # check_existence_of_scatter_files() + #filenames = list_files('scatter_spectra_files') + p = np.zeros(len(gases)) + p[0:-1] = scatter_proportion + p[-1] = 1 - sum(scatter_proportion) + scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') + scatter_spectra = np.load( + scatter_spectra_file_path, allow_pickle = True + ) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += current_working_spectrum + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(len(self.gases))): + coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M + current_full_spectrum += coefficient*current_working_spectrum + return current_full_spectrum + + def spectrum_func_ftc_2(self, bins_Hz, *p0): + B_field = p0[0] + amplitude = p0[1] + prob_parameter = p0[2] + N = len(self.gases) + scatter_proportion = p0[3:2+N] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + en_array_rev = ComplexLineShapeUtilities.flip_array(-1*en_loss_array) + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_ftc_2(prob_parameter, scatter_proportion) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum) + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + return f + + def fit_data_ftc_2(self, freq_bins, data_hist_freq): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + bins_Hz_nonzero , data_hist_nonzero , data_hist_err = ComplexLineShapeUtilities.get_only_nonzero_bins(bins_Hz, data_hist_freq) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + N = len(self.gases) + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess] + (N-1)*[scatter_proportion_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max)] + (N-1)*[(scatter_proportion_min, scatter_proportion_max)] + logger.info(p0_guess) + logger.info(p0_bounds) + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi2_Poisson(bins_Hz, data_hist_freq, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + # Name each of the resulting parameters and errors + ################### Generalize to N Gases ########################### + B_field_fit = params[0] + amplitude_fit = params[1] + prob_parameter_fit = params[2] + scatter_proportion_fit = list(params[3:2+N])+[1- sum(params[3:2+N])] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + prob_parameter_fit_err = perr[2] + scatter_proportion_fit_err = list(perr[3:2+N])+[np.sqrt(sum(perr[3:2+N]**2))] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_ftc_2(bins_Hz, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + + reduced_chi2 = self.reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = 4) + elapsed = time.time() - t + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)+' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' + output_string += '-----------------\n' + output_string += '' + for i in range(len(self.gases)): + output_string += '{} Scatter proportion \n= '.format(self.gases[i]) + "{:.8e}".format(scatter_proportion_fit[i])\ + +' +/- ' + "{:.2e}".format(scatter_proportion_fit_err[i])+'\n' + output_string += '-----------------\n' + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'prob_parameter_fit': prob_parameter_fit, + 'prob_parameter_fit_err': prob_parameter_fit_err, + 'scatter_proportion_fit': scatter_proportion_fit, + 'scatter_proportion_fit_err': scatter_proportion_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results From 7c013dc51f00d9abbe9eecd5603f206a7bcccba2 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Sun, 18 Oct 2020 19:42:07 -0400 Subject: [PATCH 023/199] np.random in convolve_ins_resolution --- .../misc/MultiGasComplexLineShape.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index ea7a4af0..919d20a7 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -120,6 +120,20 @@ def std_lorenztian_17keV(self): ans = lorentzian(x_array,0,kr_line_width) return ans + #A Dirac delta functin + def std_dirac(self): + x_array = self.std_eV_array() + ans = np.zeros(len(x_array)) + min_x = np.min(np.abs(x_array)) + ans[np.abs(x_array)==min_x] = 1. + logger.warning('Spectrum will be shifted by lineshape by {} eV'.format(min_x)) + if min_x > 0.1: + logger.warning('Lineshape will shift spectrum by > 0.1 eV') + if min_x > 1.: + logger.warning('Lineshape will shift spectrum by > 1 eV') + raise ValueError('problem with std_eV_array()') + return ans + # A gaussian function def gaussian(self, x_array, A, sigma, mu): f = A*(1./(sigma*np.sqrt(2*np.pi)))*np.exp(-(((x_array-mu)/sigma)**2.)/2.) @@ -289,7 +303,9 @@ def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): return x_data, y_data, y_err_data def convolve_ins_resolution(self, working_spectrum): - x_data, y_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + x_data, y_mean_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + y_data = np.random.normal(y_mean_data, y_err_data) + y_data[y_data<0] = 0 f = interpolate.interp1d(x_data, y_data) x_array = self.std_eV_array() y_array = np.zeros(len(x_array)) @@ -694,6 +710,8 @@ def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + elif emitted_peak == 'dirac': + current_working_spectrum = self.std_dirac() current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum From cc4f411a6b2b6ffce52f0b3806fcaff0f05c82c3 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 19 Oct 2020 17:32:04 -0400 Subject: [PATCH 024/199] update convolve_ins_resolution_combining_four_trap --- .../misc/MultiGasComplexLineShape.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 919d20a7..5f03a83a 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -67,6 +67,7 @@ def InternalConfigure(self, params): self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') self.path_to_missing_track_radiation_loss_data_numpy_file = '/host/' self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all4.txt') + self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/host/res_all_conversion_max25_trap1.txt', 'res_all_conversion_max25_trap2.txt', 'res_all_conversion_max25_trap3.txt', 'res_all_conversion_max25_trap4.txt']) if not os.path.exists(self.shake_spectrum_parameters_json_path): raise IOError('Shake spectrum path does not exist') @@ -314,7 +315,23 @@ def convolve_ins_resolution(self, working_spectrum): convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum - + + def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_array): + y_data_array = [] + y_err_data_array = [] + for path_to_single_trap_resolution_txt in self.path_to_four_trap_ins_resolution_data_txt: + x_data, y_data, y_err_data = read_ins_resolution_data(self, path_to_single_trap_resolution_txt) + y_data_array.append(y_data) + y_err_data_array.append(y_err_data) + y_data_combined = weight_array[0]*y_data_array[0] + weight_array[1]*y_data_array[1] + weight_array[2]*y_data_array[2] + weight_array[3]*x_data_array[3] + y_err_data_combined = np.sqrt((weight_array[0]*y_data_array[0])**2 + (weight_array[1]*y_data_array[1])**2 + (weight_array[2]*y_data_array[2])**2 + (weight_array[3]*x_data_array[3])**2) + x_array = self.std_eV_array() + y_array = np.zeros(len(x_array)) + index_within_range_of_xdata = np.where((x_array >= x_data[0]) & (x_array <= x_data[-1])) + y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]) + convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') + normalized_convolved_spectrum = self.normalize(convolved_spectrum) + def least_square(self, bin_centers, hist, params): # expectation expectation = self.spectrum_func_ftc(bin_centers, *params) From bd27aaab866b6e18c3004c0a62f1c5c4473571a4 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 19 Oct 2020 22:02:13 -0400 Subject: [PATCH 025/199] update convolve_ins_resolution_combining_four_trap --- mermithid/processors/misc/MultiGasComplexLineShape.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 5f03a83a..dad2d211 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -315,8 +315,8 @@ def convolve_ins_resolution(self, working_spectrum): convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum - - def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_array): + + def combine_four_trap_resolution_from_txt(weight_array): y_data_array = [] y_err_data_array = [] for path_to_single_trap_resolution_txt in self.path_to_four_trap_ins_resolution_data_txt: @@ -325,12 +325,18 @@ def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_a y_err_data_array.append(y_err_data) y_data_combined = weight_array[0]*y_data_array[0] + weight_array[1]*y_data_array[1] + weight_array[2]*y_data_array[2] + weight_array[3]*x_data_array[3] y_err_data_combined = np.sqrt((weight_array[0]*y_data_array[0])**2 + (weight_array[1]*y_data_array[1])**2 + (weight_array[2]*y_data_array[2])**2 + (weight_array[3]*x_data_array[3])**2) + return x_data, y_data_combined, y_err_data_combined + + def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_array): + x_data, y_data_combined, y_err_data_combined = self.combine_four_trap_resolution_from_txt(weight_array) + f = interpolate.interp1d(x_data, y_data_combined) x_array = self.std_eV_array() y_array = np.zeros(len(x_array)) index_within_range_of_xdata = np.where((x_array >= x_data[0]) & (x_array <= x_data[-1])) y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]) convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') normalized_convolved_spectrum = self.normalize(convolved_spectrum) + return normalized_convolved_spectrum def least_square(self, bin_centers, hist, params): # expectation From acf830ae89f61a0aafb4cbe3aed278a171b5415a Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Mon, 19 Oct 2020 22:42:28 -0400 Subject: [PATCH 026/199] started adding new options re ins resolution --- mermithid/processors/misc/MultiGasComplexLineShape.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 5f03a83a..f92c3acd 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -57,6 +57,8 @@ def InternalConfigure(self, params): self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) self.fit_ftc = reader.read_param(params, 'fit_ftc', True) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', False) + self.sample_ins_resolution_errors = reader.read_param(params, 'sample_in_errors', False) + self.combine_ins_res_files = reader.read_param(params, 'combine_ins_files', False) # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown self.num_points_in_std_array = reader.read_param(params, 'num_points_in_std_array', 10000) @@ -305,7 +307,10 @@ def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): def convolve_ins_resolution(self, working_spectrum): x_data, y_mean_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) - y_data = np.random.normal(y_mean_data, y_err_data) + if self.sample_ins_resolution_errors: + y_data = np.random.normal(y_mean_data, y_err_data) + else: + y_data = y_mean_data y_data[y_data<0] = 0 f = interpolate.interp1d(x_data, y_data) x_array = self.std_eV_array() From 3231065322a6adba29c7198a59abef6cf4bc0d05 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 19 Oct 2020 22:44:46 -0400 Subject: [PATCH 027/199] add dirac peak option to every make_spectrum --- mermithid/processors/misc/MultiGasComplexLineShape.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index dad2d211..a3a8609a 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -408,6 +408,8 @@ def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitt current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + elif emitted_peak == 'dirac': + current_working_spectrum = self.std_dirac() current_working_spectrum = self.convolve_gaussian(current_working_spectrum, gauss_FWHM_eV) zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum @@ -578,6 +580,8 @@ def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak='shake'): current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + elif emitted_peak == 'dirac': + current_working_spectrum = self.std_dirac() current_working_spectrum = self.convolve_gaussian(current_working_spectrum, gauss_FWHM_eV) zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum @@ -872,6 +876,8 @@ def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak=' current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + elif emitted_peak == 'dirac': + current_working_spectrum = self.std_dirac() current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum From 0d6cd30825b9ab2adfa4e95c53d0e7fea48e9d32 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 19 Oct 2020 22:48:17 -0400 Subject: [PATCH 028/199] change fix_ftc to use_simulated_inst_reso --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 1 + mermithid/processors/misc/MultiGasComplexLineShape.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index f0ef3a77..f84448e2 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -152,6 +152,7 @@ def InternalConfigure(self, params): 'fix_scatter_proportion': True, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below 'gas1_scatter_proportion': self.scatter_proportion, + 'use_simulated_inst_reso': True # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to # be increased for larger datasets. diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index a3a8609a..019775e4 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -55,7 +55,7 @@ def InternalConfigure(self, params): self.fixed_scatter_proportion = reader.read_param(params, 'fixed_scatter_proportion', True) if self.fixed_scatter_proportion == True: self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) - self.fit_ftc = reader.read_param(params, 'fit_ftc', True) + self.use_simulated_inst_reso = reader.read_param(params, 'use_simulated_inst_reso', True) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', False) # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown @@ -93,12 +93,12 @@ def InternalRun(self): # kr17kev_in_hz = guess*(bins[1]-bins[0])+bins[0] #self.B_field = B(17.8, kr17kev_in_hz + 0) if self.fixed_scatter_proportion == True: - if self.fit_ftc == True: + if self.use_simulated_inst_reso == True: self.results = self.fit_data_ftc(freq_bins, data_hist_freq) else: self.results = self.fit_data_1(freq_bins, data_hist_freq) else: - if self.fit_ftc == True: + if self.use_simulated_inst_reso == True: self.results = self.fit_data_ftc_2(freq_bins, data_hist_freq) else: self.results = self.fit_data(freq_bins, data_hist_freq) From 33dece6744aa123f56e92bf5d11764981e960eb9 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 19 Oct 2020 22:55:50 -0400 Subject: [PATCH 029/199] add configuration use_combined_four_trap_inst_reso --- mermithid/processors/misc/MultiGasComplexLineShape.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 019775e4..28e33789 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -67,6 +67,7 @@ def InternalConfigure(self, params): self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') self.path_to_missing_track_radiation_loss_data_numpy_file = '/host/' self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all4.txt') + self.use_combined_four_trap_inst_reso = reader.read_param(params, 'use_combined_four_trap_inst_reso', False) self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/host/res_all_conversion_max25_trap1.txt', 'res_all_conversion_max25_trap2.txt', 'res_all_conversion_max25_trap3.txt', 'res_all_conversion_max25_trap4.txt']) if not os.path.exists(self.shake_spectrum_parameters_json_path): @@ -739,7 +740,11 @@ def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() elif emitted_peak == 'dirac': current_working_spectrum = self.std_dirac() - current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) + if use_combined_four_trap_inst_reso: + weight_array = [] + current_working_spectrum = self.convolve_ins_resolution_combining_four_trap(current_working_spectrum) + else: + current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum N = len(self.gases) From 5474e89f48373e6d4738bb7091e5e0c37721a346 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 19 Oct 2020 22:57:49 -0400 Subject: [PATCH 030/199] add configuration use_combined_four_trap_inst_reso in FakeDataGenerator processor --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index f84448e2..80e3d8d4 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -153,6 +153,7 @@ def InternalConfigure(self, params): # When fix_scatter_proportion is True, set the scatter proportion for gas1 below 'gas1_scatter_proportion': self.scatter_proportion, 'use_simulated_inst_reso': True + 'use_combined_four_trap_inst_reso': False # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to # be increased for larger datasets. From a9b1ac7a33b6a0edb6b1a041a37e8fbb465b990b Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Mon, 19 Oct 2020 23:06:37 -0400 Subject: [PATCH 031/199] Continued YuHao's work adding trap combining/error sampling --- .../TritiumSpectrum/FakeDataGenerator.py | 9 ++++---- .../misc/MultiGasComplexLineShape.py | 23 +++++++++++++++---- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index f0ef3a77..2da55ce6 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -16,7 +16,7 @@ from morpho.utilities import morphologging, reader from morpho.processors import BaseProcessor from mermithid.misc.FakeTritiumDataFunctions import * -from mermithid.processors.misc.KrComplexLineShape import KrComplexLineShape +from mermithid.processors.misc.MultiGasComplexLineShape import MultiGasComplexLineShape from mermithid.misc import Constants, ComplexLineShapeUtilities, ConversionFunctions logger = morphologging.getLogger(__name__) @@ -158,11 +158,12 @@ def InternalConfigure(self, params): 'num_points_in_std_array': 10000, 'B_field': self.B_field, 'base_shape': 'dirac', - 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path, - 'path_to_ins_resolution_data_txt': self.path_to_ins_resolution_data_txt + 'sample_ins_res_errors': True, + 'combine_ins_res_files': True, + 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path } logger.info('Setting up complex lineshape object') - self.complexLineShape = KrComplexLineShape("complexLineShape") + self.complexLineShape = MultiGasComplexLineShape("complexLineShape") logger.info('Configuring complex lineshape') self.complexLineShape.Configure(complexLineShape_config) logger.info('Checking existence of scatter spectra files') diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 05ae9ec2..62e6455a 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -52,13 +52,14 @@ def InternalConfigure(self, params): self.bins_choice = reader.read_param(params, 'bins_choice', []) self.gases = reader.read_param(params, 'gases', ["H2", "Kr", "He", "Ar"]) self.max_scatters = reader.read_param(params, 'max_scatters', 20) + self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.01, 0.01, 0.01, 0.01]}) #Weights from Xueying's Sept. 13 slides; errors currently arbitrary self.fixed_scatter_proportion = reader.read_param(params, 'fixed_scatter_proportion', True) if self.fixed_scatter_proportion == True: self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) self.fit_ftc = reader.read_param(params, 'fit_ftc', True) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', False) - self.sample_ins_resolution_errors = reader.read_param(params, 'sample_in_errors', False) - self.combine_ins_res_files = reader.read_param(params, 'combine_ins_files', False) + self.sample_ins_resolution_errors = reader.read_param(params, 'sample_ins_res_errors', False) + self.combine_ins_resolution_files = reader.read_param(params, 'combine_ins_res_files', False) # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown self.num_points_in_std_array = reader.read_param(params, 'num_points_in_std_array', 10000) @@ -321,7 +322,11 @@ def convolve_ins_resolution(self, working_spectrum): normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum - def combine_four_trap_resolution_from_txt(weight_array): + def combine_four_trap_resolution_from_txt(trap_weights): + if self.sample_ins_resolution_errors: + weight_array = np.random.normal(trap_weights['weights'], trap_weights['errors']) + else: + weight_array = trap_weights['weights'] y_data_array = [] y_err_data_array = [] for path_to_single_trap_resolution_txt in self.path_to_four_trap_ins_resolution_data_txt: @@ -334,6 +339,8 @@ def combine_four_trap_resolution_from_txt(weight_array): def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_array): x_data, y_data_combined, y_err_data_combined = self.combine_four_trap_resolution_from_txt(weight_array) + if self.sample_ins_resolution_errors: + y_data_combined = np.random.normal(y_data_combined, y_err_data) f = interpolate.interp1d(x_data, y_data_combined) x_array = self.std_eV_array() y_array = np.zeros(len(x_array)) @@ -740,7 +747,10 @@ def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() elif emitted_peak == 'dirac': current_working_spectrum = self.std_dirac() - current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) + if self.combine_ins_resolution_files: + current_working_spectrum = self.convolve_ins_resolution_combining_four_trap(current_working_spectrum, self.trap_weights) + else: + current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum N = len(self.gases) @@ -877,7 +887,10 @@ def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak=' current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() - current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) + if self.combine_ins_resolution_files: + current_working_spectrum = self.convolve_ins_resolution_combining_four_trap(current_working_spectrum, self.trap_weights) + else: + current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum N = len(self.gases) From d3496c983c17ea384061ffd6a558c59a55ca141f Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Mon, 19 Oct 2020 23:19:40 -0400 Subject: [PATCH 032/199] Set radiation loss to True by default --- mermithid/processors/misc/MultiGasComplexLineShape.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 8599bc1f..490023ad 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -57,7 +57,7 @@ def InternalConfigure(self, params): if self.fixed_scatter_proportion == True: self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) self.use_simulated_inst_reso = reader.read_param(params, 'use_simulated_inst_reso', True) - self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', False) + self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) self.sample_ins_resolution_errors = reader.read_param(params, 'sample_ins_res_errors', False) # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown From 673bd510c78dba379052dd1c7a42c03baf61f38c Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Tue, 20 Oct 2020 08:48:12 -0400 Subject: [PATCH 033/199] add base_shape configuration to multi gas line shape processor --- mermithid/processors/misc/MultiGasComplexLineShape.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 490023ad..ece9da03 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -64,6 +64,7 @@ def InternalConfigure(self, params): self.num_points_in_std_array = reader.read_param(params, 'num_points_in_std_array', 10000) self.RF_ROI_MIN = reader.read_param(params, 'RF_ROI_MIN', 25850000000.0) self.B_field = reader.read_param(params, 'B_field', 0.957810722501) + self.base_shape = reader.read_param(params, 'base_shape', 'dirac') self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') @@ -401,7 +402,7 @@ def reduced_chi2_Poisson(self, data_hist_freq, fit_Hz, number_of_parameters): reduced_chi2 = chi2/(len(data_hist_freq) - number_of_parameters) return reduced_chi2 - def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitted_peak='shake'): + def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitted_peak=self.base_shape): gases = self.gases max_scatters = self.max_scatters current_path = self.path_to_scatter_spectra_file @@ -575,7 +576,7 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): } return dictionary_of_fit_results - def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak='shake'): + def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak=self.base_shape): gases = self.gases current_path = self.path_to_scatter_spectra_file # check_existence_of_scatter_files() @@ -732,7 +733,7 @@ def fit_data_1(self, freq_bins, data_hist_freq): } return dictionary_of_fit_results - def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): + def make_spectrum_ftc(self, prob_parameter, emitted_peak=self.base_shape): gases = self.gases current_path = self.path_to_scatter_spectra_file # check_existence_of_scatter_files() @@ -873,7 +874,7 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): return dictionary_of_fit_results - def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak='shake'): + def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak=self.base_shape): gases = self.gases current_path = self.path_to_scatter_spectra_file # check_existence_of_scatter_files() From c03617e9e43918034bc3183c95aea4d5de8649c1 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Tue, 20 Oct 2020 10:01:24 -0400 Subject: [PATCH 034/199] Make B-field fixed input for fake data gen --- mermithid/misc/FakeTritiumDataFunctions.py | 9 ++++----- .../processors/TritiumSpectrum/FakeDataGenerator.py | 7 +++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index a8c6cb5e..56b72a87 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -244,7 +244,7 @@ def convolved_bkgd_rate(K, Kmin, Kmax, lineshape, ls_params, min_energy, max_ene #Convolution of signal and lineshape using scipy.signal.convolve def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, - lineshape, ls_params, min_energy, max_energy, complexLineShape): + lineshape, ls_params, min_energy, max_energy, B_field, complexLineShape): """K is an array-like object """ logger.info('Using scipy convolve') @@ -262,8 +262,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - - lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., 0, 1, ls_params[1]) + lineshape_rates = complexLineShape.spectrum_func_ftc(K_lineshape/1000., B_field, 1, ls_params[1]) beta_rates = np.zeros(len(K)) for i,ke in enumerate(K): @@ -278,7 +277,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, #Convolution of background and lineshape using scipy.signal.convolve -def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, max_energy, complexLineShape): +def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, max_energy, B_field, complexLineShape): """K is an array-like object """ energy_half_range = max(max_energy, abs(min_energy)) @@ -294,7 +293,7 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., 0, 1, ls_params[1]) + lineshape_rates = complexLineShape.spectrum_func_ftc(K_lineshape/1000., B_field, 1, ls_params[1]) bkgd_rates = np.full(len(K), bkgd_rate()) if len(K) < len(K_lineshape): diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index e2a334a6..4f6e541e 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -158,7 +158,6 @@ def InternalConfigure(self, params): # the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to # be increased for larger datasets. 'num_points_in_std_array': 10000, - 'B_field': self.B_field, 'base_shape': 'dirac', 'sample_ins_res_errors': True, 'use_combined_four_trap_inst_reso': True, @@ -252,7 +251,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 - 'gaussian' - 'simplified': Central gaussian + approximated scattering - 'detailed': Central gaussian + detailed scattering - 'params' is a list of the params inputted into the lineshape function. The first entry of the list should be a standard deviation of full width half max that provides the scale of the lineshape width. + 'params' is a list of the params inputted into the lineshape function. If such a parameter exists, the first entry of the list should be a standard deviation of full width half max that provides the scale of the lineshape width. """ logger.info('Going to generate pseudo-unbinned data with {} lineshape'.format(lineshape)) @@ -305,7 +304,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 time0 = time.time() if array_method == True: - ratesS = convolved_spectral_rate_arrays(self.Koptions, Q_mean, mass, Kmin, lineshape, params, min_energy, max_energy, self.complexLineShape) + ratesS = convolved_spectral_rate_arrays(self.Koptions, Q_mean, mass, Kmin, lineshape, params, min_energy, max_energy, B_field, self.complexLineShape) else: ratesS = [convolved_spectral_rate(K, Q_mean, mass, Kmin, lineshape, params, min_energy, max_energy) for K in self.Koptions] @@ -317,7 +316,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 # background if array_method == True: - ratesB = convolved_bkgd_rate_arrays(self.Koptions, Kmin, Kmax, lineshape, params, min_energy, max_energy, self.complexLineShape) + ratesB = convolved_bkgd_rate_arrays(self.Koptions, Kmin, Kmax, lineshape, params, min_energy, max_energy, B_field, self.complexLineShape) else: ratesB = [convolved_bkgd_rate(K, Kmin, Kmax, lineshape, params, min_energy, max_energy) for K in self.Koptions] From 726b88d0c2192ec2ab600d628e2d9b9799b9cf15 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Tue, 20 Oct 2020 18:54:24 -0400 Subject: [PATCH 035/199] Added missing commas --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 4f6e541e..ea603e75 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -152,8 +152,8 @@ def InternalConfigure(self, params): 'fix_scatter_proportion': True, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below 'gas1_scatter_proportion': self.scatter_proportion, - 'use_simulated_inst_reso': True - 'use_combined_four_trap_inst_reso': False + 'use_simulated_inst_reso': True, + 'use_combined_four_trap_inst_reso': False, # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to # be increased for larger datasets. From fd5eead44712f5ef5ccf394201b14ad7ded7a930 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Tue, 20 Oct 2020 19:54:11 -0400 Subject: [PATCH 036/199] put emitted_peak = self.base_shape inside make_spectrum functions --- .../processors/misc/MultiGasComplexLineShape.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index ece9da03..d0cc1904 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -402,7 +402,7 @@ def reduced_chi2_Poisson(self, data_hist_freq, fit_Hz, number_of_parameters): reduced_chi2 = chi2/(len(data_hist_freq) - number_of_parameters) return reduced_chi2 - def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitted_peak=self.base_shape): + def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitted_peak='shake'): gases = self.gases max_scatters = self.max_scatters current_path = self.path_to_scatter_spectra_file @@ -417,6 +417,7 @@ def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitt ) en_array = self.std_eV_array() current_full_spectrum = np.zeros(len(en_array)) + emitted_peak = self.base_shape if emitted_peak == 'lorentzian': current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': @@ -576,7 +577,7 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): } return dictionary_of_fit_results - def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak=self.base_shape): + def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak='shake'): gases = self.gases current_path = self.path_to_scatter_spectra_file # check_existence_of_scatter_files() @@ -589,6 +590,7 @@ def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak=self.base_ ) en_array = self.std_eV_array() current_full_spectrum = np.zeros(len(en_array)) + emitted_peak = self.base_shape if emitted_peak == 'lorentzian': current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': @@ -733,7 +735,7 @@ def fit_data_1(self, freq_bins, data_hist_freq): } return dictionary_of_fit_results - def make_spectrum_ftc(self, prob_parameter, emitted_peak=self.base_shape): + def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): gases = self.gases current_path = self.path_to_scatter_spectra_file # check_existence_of_scatter_files() @@ -746,6 +748,7 @@ def make_spectrum_ftc(self, prob_parameter, emitted_peak=self.base_shape): ) en_array = self.std_eV_array() current_full_spectrum = np.zeros(len(en_array)) + emitted_peak = self.base_shape if emitted_peak == 'lorentzian': current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': @@ -874,7 +877,7 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): return dictionary_of_fit_results - def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak=self.base_shape): + def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak='shake'): gases = self.gases current_path = self.path_to_scatter_spectra_file # check_existence_of_scatter_files() @@ -888,6 +891,7 @@ def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak=s ) en_array = self.std_eV_array() current_full_spectrum = np.zeros(len(en_array)) + emitted_peak = self.baseshape if emitted_peak == 'lorentzian': current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': From 684ca43aa3bcd51d0569a178056f865fdcd78875 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Tue, 20 Oct 2020 22:22:45 -0400 Subject: [PATCH 037/199] fixed various small errors in implementation of new features --- .../TritiumSpectrum/FakeDataGenerator.py | 3 ++- .../misc/MultiGasComplexLineShape.py | 20 ++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index ea603e75..98b43d4b 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -161,7 +161,8 @@ def InternalConfigure(self, params): 'base_shape': 'dirac', 'sample_ins_res_errors': True, 'use_combined_four_trap_inst_reso': True, - 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path + 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path, + 'path_to_scatter_spectra_file':self.detailed_scatter_spectra_path } logger.info('Setting up complex lineshape object') self.complexLineShape = MultiGasComplexLineShape("complexLineShape") diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index d0cc1904..e67f8c96 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -52,7 +52,7 @@ def InternalConfigure(self, params): self.bins_choice = reader.read_param(params, 'bins_choice', []) self.gases = reader.read_param(params, 'gases', ["H2", "Kr", "He", "Ar"]) self.max_scatters = reader.read_param(params, 'max_scatters', 20) - self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.01, 0.01, 0.01, 0.01]}) #Weights from Xueying's Sept. 13 slides; errors currently arbitrary + self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.003, 0.013, 0.014, 0.02]}) #Weights from Xueying's Sept. 13 slides; errors currently arbitrary self.fixed_scatter_proportion = reader.read_param(params, 'fixed_scatter_proportion', True) if self.fixed_scatter_proportion == True: self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) @@ -68,12 +68,12 @@ def InternalConfigure(self, params): self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') - self.path_to_missing_track_radiation_loss_data_numpy_file = '/host/' + self.path_to_missing_track_radiation_loss_data_numpy_file = reader.read_param(params, 'rad_loss_path', '/termite/analysis_input/complex-lineshape-inputs') self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all4.txt') self.use_combined_four_trap_inst_reso = reader.read_param(params, 'use_combined_four_trap_inst_reso', False) - self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/host/res_all_conversion_max25_trap1.txt', 'res_all_conversion_max25_trap2.txt', 'res_all_conversion_max25_trap3.txt', 'res_all_conversion_max25_trap4.txt']) + self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max25_trap1.txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max25_trap2.txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max25_trap3.txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max25_trap4.txt']) - if not os.path.exists(self.shake_spectrum_parameters_json_path): + if not os.path.exists(self.shake_spectrum_parameters_json_path) and self.base_shape=='shake': raise IOError('Shake spectrum path does not exist') if not os.path.exists(self.path_to_osc_strengths_files): raise IOError('Path to osc strengths files does not exist') @@ -323,7 +323,7 @@ def convolve_ins_resolution(self, working_spectrum): normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum - def combine_four_trap_resolution_from_txt(trap_weights): + def combine_four_trap_resolution_from_txt(self, trap_weights): if self.sample_ins_resolution_errors: weight_array = np.random.normal(trap_weights['weights'], trap_weights['errors']) else: @@ -331,17 +331,17 @@ def combine_four_trap_resolution_from_txt(trap_weights): y_data_array = [] y_err_data_array = [] for path_to_single_trap_resolution_txt in self.path_to_four_trap_ins_resolution_data_txt: - x_data, y_data, y_err_data = read_ins_resolution_data(self, path_to_single_trap_resolution_txt) + x_data, y_data, y_err_data = self.read_ins_resolution_data(path_to_single_trap_resolution_txt) y_data_array.append(y_data) y_err_data_array.append(y_err_data) - y_data_combined = weight_array[0]*y_data_array[0] + weight_array[1]*y_data_array[1] + weight_array[2]*y_data_array[2] + weight_array[3]*x_data_array[3] - y_err_data_combined = np.sqrt((weight_array[0]*y_data_array[0])**2 + (weight_array[1]*y_data_array[1])**2 + (weight_array[2]*y_data_array[2])**2 + (weight_array[3]*x_data_array[3])**2) + y_data_combined = weight_array[0]*y_data_array[0] + weight_array[1]*y_data_array[1] + weight_array[2]*y_data_array[2] + weight_array[3]*y_data_array[3] + y_err_data_combined = np.sqrt((weight_array[0]*y_data_array[0])**2 + (weight_array[1]*y_data_array[1])**2 + (weight_array[2]*y_data_array[2])**2 + (weight_array[3]*y_data_array[3])**2) return x_data, y_data_combined, y_err_data_combined def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_array): x_data, y_data_combined, y_err_data_combined = self.combine_four_trap_resolution_from_txt(weight_array) if self.sample_ins_resolution_errors: - y_data_combined = np.random.normal(y_data_combined, y_err_data) + y_data_combined = np.random.normal(y_data_combined, y_err_data_combined) f = interpolate.interp1d(x_data, y_data_combined) x_array = self.std_eV_array() y_array = np.zeros(len(x_array)) @@ -772,6 +772,8 @@ def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): current_working_spectrum = scatter_spectra.item()[entry_str] current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) coefficient = factorial(sum(combination)) + print(len(self.gases)) + print(len(p)) for component, i in zip(combination, range(len(self.gases))): coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M current_full_spectrum += coefficient*current_working_spectrum From ab86d1d960781df0a90598aab5c11cee7752501e Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 21 Oct 2020 10:40:34 -0400 Subject: [PATCH 038/199] prepare to add smeared triangle resolutio and gaussian+lorentzian resolution --- .../misc/MultiGasComplexLineShape.py | 25 ++++++++++++------- test_analysis/Complex_line_shape_fitter.py | 11 ++++---- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index dae741a1..ea7a4af0 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -66,7 +66,7 @@ def InternalConfigure(self, params): self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') self.path_to_missing_track_radiation_loss_data_numpy_file = '/host/' - self.path_to_ins_resolution_data_txt = '/host/' + self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all4.txt') if not os.path.exists(self.shake_spectrum_parameters_json_path): raise IOError('Shake spectrum path does not exist') @@ -98,7 +98,7 @@ def InternalRun(self): self.results = self.fit_data_1(freq_bins, data_hist_freq) else: if self.fit_ftc == True: - self.results = self.fit_data_ftc_1(freq_bins, data_hist_freq) + self.results = self.fit_data_ftc_2(freq_bins, data_hist_freq) else: self.results = self.fit_data(freq_bins, data_hist_freq) @@ -180,7 +180,7 @@ def another_scatter(self, input_spectrum, gas_type): def radiation_loss_f(self): radiation_loss_data_file_path = self.path_to_missing_track_radiation_loss_data_numpy_file + '/missing_track_radiation_loss.npy' data_for_missing_track_radiation_loss = np.load(radiation_loss_data_file_path, allow_pickle = True) - x_data_for_histogram = f_radiation_energy_loss_interp = data_for_missing_track_radiation_loss.item()['histogram_eV']['x_data'] + x_data_for_histogram = data_for_missing_track_radiation_loss.item()['histogram_eV']['x_data'] energy_loss_array = self.std_eV_array() f_radiation_energy_loss = 0 * energy_loss_array f_radiation_energy_loss_interp = data_for_missing_track_radiation_loss.item()['histogram_eV']['interp'] @@ -282,7 +282,7 @@ def convolve_composite_gaussian(self, func_to_convolve, A_array, sigma_array): return ans_normed def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): - ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt+'ins_resolution_all4.txt') + ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt) x_data = ins_resolution_data.T[0] y_data = ins_resolution_data.T[1] y_err_data = ins_resolution_data.T[2] @@ -316,10 +316,16 @@ def chi2_Poisson(self, bin_centers, data_hist_freq, params): nonzero_bins_index = np.where(data_hist_freq != 0) zero_bins_index = np.where(data_hist_freq == 0) # expectation - if self.fit_ftc: - fit_Hz = self.spectrum_func_ftc(bin_centers, *params) + if self.fixed_scatter_proportion: + if self.fit_ftc: + fit_Hz = self.spectrum_func_ftc(bin_centers, *params) + else: + fit_Hz = self.spectrum_func_1(bin_centers, *params) else: - fit_Hz = self.spectrum_func_1(bin_centers, *params) + if self.fit_ftc: + fit_Hz = self.spectrum_func_ftc_2(bin_centers, *params) + else: + fit_Hz = self.spectrum_func(bin_centers, *params) chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() return chi2 @@ -863,7 +869,7 @@ def spectrum_func_ftc_2(self, bins_Hz, *p0): zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - full_spectrum = self.make_spectrum_ftc_1(prob_parameter, scatter_proportion) + full_spectrum = self.make_spectrum_ftc_2(prob_parameter, scatter_proportion) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum) f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) return f @@ -916,7 +922,7 @@ def fit_data_ftc_2(self, freq_bins, data_hist_freq): scatter_proportion_fit_err = list(perr[3:2+N])+[np.sqrt(sum(perr[3:2+N]**2))] total_counts_fit_err = amplitude_fit_err - fit_Hz = self.spectrum_func_ftc_1(bins_Hz, *params) + fit_Hz = self.spectrum_func_ftc_2(bins_Hz, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) @@ -932,6 +938,7 @@ def fit_data_ftc_2(self, freq_bins, data_hist_freq): output_string += '-----------------\n' output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)+' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' output_string += '-----------------\n' + output_string += '' for i in range(len(self.gases)): output_string += '{} Scatter proportion \n= '.format(self.gases[i]) + "{:.8e}".format(scatter_proportion_fit[i])\ +' +/- ' + "{:.2e}".format(scatter_proportion_fit_err[i])+'\n' diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 0f89c016..fe178066 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -29,12 +29,12 @@ def test_complex_lineshape(self): } complexLineShape_config = { 'bins_choice': np.linspace(0e6, 100e6, 1000), - 'gases': ["H2", "Ar"], + 'gases': ["H2", "Kr"], 'max_scatters': 20, - 'fixed_scatter_proportion': True, + 'fixed_scatter_proportion': False, 'fit_ftc':True, # use gaussian instrumental resolution # When fix_scatter_proportion is True, set the scatter proportion for gas1 below - 'gas_scatter_proportion': [0.93, 0.02], + 'gas_scatter_proportion': [0.61, 0.34], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -42,7 +42,8 @@ def test_complex_lineshape(self): # shake_spectrum_parameters.json and oscillator strength data can be found at https://github.com/project8/scripts/tree/master/yuhao/line_shape_fitting/data 'shake_spectrum_parameters_json_path': '../mermithid/misc/shake_spectrum_parameters.json', 'path_to_osc_strengths_files': '/host/', - 'path_to_scatter_spectra_file': '/host/' + 'path_to_scatter_spectra_file': '/host/', + 'path_to_ins_resolution_data_txt': '/host/res_all_conversion_max25.txt' } b = IOCicadaProcessor("reader") @@ -85,7 +86,7 @@ def test_complex_lineshape(self): plot_title += '\n with fixed scatter proportion \n {}'.format(str_gas_scatter_proportion) plt.title(plot_title) plt.tight_layout() - plt.savefig('/host/plots/fit_FTC_march_with_{}_gas_scattering_gaussian_ins_res.png'.format(len(complexLineShape_config['gases']))) + plt.savefig('/host/plots/fit_FTC_march_with_{}_gas_scattering_max25_fitting_H2_He.png'.format(len(complexLineShape_config['gases']))) if __name__ == '__main__': From 2871962e603c776ca6b0df340e937106a2755624 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Wed, 21 Oct 2020 11:27:03 -0400 Subject: [PATCH 039/199] Fixed errors; added temporary diagnoistic plots --- mermithid/misc/FakeTritiumDataFunctions.py | 13 ++++++++++ .../TritiumSpectrum/FakeDataGenerator.py | 9 ++++--- .../misc/MultiGasComplexLineShape.py | 25 +++++++++++++------ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 56b72a87..2423a0c1 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -19,6 +19,8 @@ from mermithid.misc.Constants import * from mermithid.misc.ConversionFunctions import * +import matplotlib.pyplot as plt + """ Constants and functions used by processors/TritiumSpectrum/FakeDataGenerator.py """ @@ -263,6 +265,12 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': lineshape_rates = complexLineShape.spectrum_func_ftc(K_lineshape/1000., B_field, 1, ls_params[1]) + #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., 0, 1, ls_params[1]) + plt.plot(K_lineshape/1000., lineshape_rates) + plt.xlabel('Energy shift (eV)') + plt.ylabel('Complex lineshape rate') + plt.savefig('complex_lineshape_rates.pdf') + plt.show() beta_rates = np.zeros(len(K)) for i,ke in enumerate(K): @@ -270,6 +278,10 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, #Convolving convolved = convolve(beta_rates, lineshape_rates, mode='same') + plt.plot(K, convolved) + plt.xlabel('Energy (eV)') + plt.ylabel('Signal rate') + plt.savefig('spectrum_signal.pdf') below_Kmin = np.where(K < Kmin) np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) return convolved @@ -294,6 +306,7 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': lineshape_rates = complexLineShape.spectrum_func_ftc(K_lineshape/1000., B_field, 1, ls_params[1]) + #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., 0, 1, ls_params[1]) bkgd_rates = np.full(len(K), bkgd_rate()) if len(K) < len(K_lineshape): diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 98b43d4b..8126e7fe 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -149,9 +149,9 @@ def InternalConfigure(self, params): complexLineShape_config = { 'gases': ["H2","He"], 'max_scatters': self.NScatters, - 'fix_scatter_proportion': True, + 'fixed_scatter_proportion': True, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below - 'gas1_scatter_proportion': self.scatter_proportion, + 'gas_scatter_proportion': [self.scatter_proportion, 1.-self.scatter_proportion], 'use_simulated_inst_reso': True, 'use_combined_four_trap_inst_reso': False, # This is an important parameter which determines how finely resolved @@ -159,7 +159,7 @@ def InternalConfigure(self, params): # be increased for larger datasets. 'num_points_in_std_array': 10000, 'base_shape': 'dirac', - 'sample_ins_res_errors': True, + 'sample_ins_resolution_errors': True, 'use_combined_four_trap_inst_reso': True, 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path, 'path_to_scatter_spectra_file':self.detailed_scatter_spectra_path @@ -339,6 +339,9 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 ratesS[ratesS<0.] = 0. ratesB[ratesB<0.] = 0. + print(ratesS) + print(len(ratesS)) + print(ratesS[2000:2010]) rate_sumS, rate_sumB = np.sum(ratesS), np.sum(ratesB) probsS = np.array(ratesS)/rate_sumS probsB = np.array(ratesB)/rate_sumB diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index e67f8c96..964a90c5 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -34,6 +34,7 @@ from morpho.utilities import morphologging, reader from morpho.processors import BaseProcessor from mermithid.misc import Constants, ComplexLineShapeUtilities, ConversionFunctions +import matplotlib.pyplot as plt logger = morphologging.getLogger(__name__) @@ -63,8 +64,7 @@ def InternalConfigure(self, params): # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown self.num_points_in_std_array = reader.read_param(params, 'num_points_in_std_array', 10000) self.RF_ROI_MIN = reader.read_param(params, 'RF_ROI_MIN', 25850000000.0) - self.B_field = reader.read_param(params, 'B_field', 0.957810722501) - self.base_shape = reader.read_param(params, 'base_shape', 'dirac') + self.base_shape = reader.read_param(params, 'base_shape', 'shake') self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') @@ -335,18 +335,33 @@ def combine_four_trap_resolution_from_txt(self, trap_weights): y_data_array.append(y_data) y_err_data_array.append(y_err_data) y_data_combined = weight_array[0]*y_data_array[0] + weight_array[1]*y_data_array[1] + weight_array[2]*y_data_array[2] + weight_array[3]*y_data_array[3] - y_err_data_combined = np.sqrt((weight_array[0]*y_data_array[0])**2 + (weight_array[1]*y_data_array[1])**2 + (weight_array[2]*y_data_array[2])**2 + (weight_array[3]*y_data_array[3])**2) + y_err_data_combined = np.sqrt((weight_array[0]*y_err_data_array[0])**2 + (weight_array[1]*y_err_data_array[1])**2 + (weight_array[2]*y_err_data_array[2])**2 + (weight_array[3]*y_err_data_array[3])**2) return x_data, y_data_combined, y_err_data_combined def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_array): x_data, y_data_combined, y_err_data_combined = self.combine_four_trap_resolution_from_txt(weight_array) + plt.errorbar(x_data, y_data_combined, yerr=y_err_data_combined) + plt.xlabel('Energy shift (eV)') + plt.ylabel('Probability (post-combining traps)') + plt.show() + plt.savefig('combined_trap_ins_res.pdf') if self.sample_ins_resolution_errors: y_data_combined = np.random.normal(y_data_combined, y_err_data_combined) + plt.plot(x_data, y_data_combined) + plt.xlabel('Energy shift (eV)') + plt.ylabel('Probability (post-sampling)') + plt.show() + plt.savefig('sampled_combined_trap_ins_res.pdf') f = interpolate.interp1d(x_data, y_data_combined) x_array = self.std_eV_array() y_array = np.zeros(len(x_array)) index_within_range_of_xdata = np.where((x_array >= x_data[0]) & (x_array <= x_data[-1])) y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]) + plt.plot(x_array, y_array) + plt.xlabel('Energy shift (eV)') + plt.ylabel('Probability (post-interpolation)') + plt.show() + plt.savefig('interpolated_combined_trap_ins_res.pdf') convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum @@ -582,7 +597,6 @@ def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak='shake'): current_path = self.path_to_scatter_spectra_file # check_existence_of_scatter_files() #filenames = list_files('scatter_spectra_files') - p = np.zeros(len(gases)) p = self.scatter_proportion scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') scatter_spectra = np.load( @@ -740,7 +754,6 @@ def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): current_path = self.path_to_scatter_spectra_file # check_existence_of_scatter_files() #filenames = list_files('scatter_spectra_files') - p = np.zeros(len(gases)) p = self.scatter_proportion scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') scatter_spectra = np.load( @@ -772,8 +785,6 @@ def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): current_working_spectrum = scatter_spectra.item()[entry_str] current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) coefficient = factorial(sum(combination)) - print(len(self.gases)) - print(len(p)) for component, i in zip(combination, range(len(self.gases))): coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M current_full_spectrum += coefficient*current_working_spectrum From 780131623f04c29ee44895bfdf358aaf132b5246 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Wed, 21 Oct 2020 15:27:00 -0400 Subject: [PATCH 040/199] Changed default inst res files --- mermithid/processors/misc/MultiGasComplexLineShape.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 964a90c5..0a14cd4d 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -69,9 +69,9 @@ def InternalConfigure(self, params): self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') self.path_to_missing_track_radiation_loss_data_numpy_file = reader.read_param(params, 'rad_loss_path', '/termite/analysis_input/complex-lineshape-inputs') - self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all4.txt') + self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/res_all_conversion_max15.5_alltraps.txt') self.use_combined_four_trap_inst_reso = reader.read_param(params, 'use_combined_four_trap_inst_reso', False) - self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max25_trap1.txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max25_trap2.txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max25_trap3.txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max25_trap4.txt']) + self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max15.5_trap1.txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max15.5_trap2.txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max15.5_trap3.txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max15.5_trap4.txt']) if not os.path.exists(self.shake_spectrum_parameters_json_path) and self.base_shape=='shake': raise IOError('Shake spectrum path does not exist') From 3fc67728f5600ecb587aca364ac77d55caa12460 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Wed, 21 Oct 2020 16:21:12 -0400 Subject: [PATCH 041/199] Fixed temporary diagnostic plots --- mermithid/misc/FakeTritiumDataFunctions.py | 7 +++++-- mermithid/processors/misc/MultiGasComplexLineShape.py | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 2423a0c1..88364d5e 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -265,7 +265,8 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': lineshape_rates = complexLineShape.spectrum_func_ftc(K_lineshape/1000., B_field, 1, ls_params[1]) - #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., 0, 1, ls_params[1]) + #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000.,ls_params[0], 0, 1, ls_params[1]) + fig = plt.figure() plt.plot(K_lineshape/1000., lineshape_rates) plt.xlabel('Energy shift (eV)') plt.ylabel('Complex lineshape rate') @@ -278,10 +279,12 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, #Convolving convolved = convolve(beta_rates, lineshape_rates, mode='same') + fig = plt.figure() plt.plot(K, convolved) plt.xlabel('Energy (eV)') plt.ylabel('Signal rate') plt.savefig('spectrum_signal.pdf') + plt.show() below_Kmin = np.where(K < Kmin) np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) return convolved @@ -306,7 +309,7 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': lineshape_rates = complexLineShape.spectrum_func_ftc(K_lineshape/1000., B_field, 1, ls_params[1]) - #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., 0, 1, ls_params[1]) + #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., ls_params[0], 0, 1, ls_params[1]) bkgd_rates = np.full(len(K), bkgd_rate()) if len(K) < len(K_lineshape): diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 0a14cd4d..4b7d170c 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -340,28 +340,31 @@ def combine_four_trap_resolution_from_txt(self, trap_weights): def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_array): x_data, y_data_combined, y_err_data_combined = self.combine_four_trap_resolution_from_txt(weight_array) + fig = plt.figure() plt.errorbar(x_data, y_data_combined, yerr=y_err_data_combined) plt.xlabel('Energy shift (eV)') plt.ylabel('Probability (post-combining traps)') - plt.show() plt.savefig('combined_trap_ins_res.pdf') + plt.show() if self.sample_ins_resolution_errors: y_data_combined = np.random.normal(y_data_combined, y_err_data_combined) + fig = plt.figure() plt.plot(x_data, y_data_combined) plt.xlabel('Energy shift (eV)') plt.ylabel('Probability (post-sampling)') - plt.show() plt.savefig('sampled_combined_trap_ins_res.pdf') + plt.show() f = interpolate.interp1d(x_data, y_data_combined) x_array = self.std_eV_array() y_array = np.zeros(len(x_array)) index_within_range_of_xdata = np.where((x_array >= x_data[0]) & (x_array <= x_data[-1])) y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]) + fig = plt.figure() plt.plot(x_array, y_array) plt.xlabel('Energy shift (eV)') plt.ylabel('Probability (post-interpolation)') - plt.show() plt.savefig('interpolated_combined_trap_ins_res.pdf') + plt.show() convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum From f884b91abb2d44152fcc2b771d37b3f8e952ff29 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 21 Oct 2020 22:12:06 -0400 Subject: [PATCH 042/199] add smeared triangle and gaussian + lorentzian reoslution --- .../misc/MultiGasComplexLineShape.py | 392 +++++++++++++++++- 1 file changed, 390 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index ea7a4af0..02163860 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -50,7 +50,7 @@ def InternalConfigure(self, params): ''' # Read other parameters self.bins_choice = reader.read_param(params, 'bins_choice', []) - self.gases = reader.read_param(params, 'gases', ["H2", "Kr", "He", "Ar"]) + self.gases = reader.read_param(params, 'gases', ["H2", "He"]) self.max_scatters = reader.read_param(params, 'max_scatters', 20) self.fixed_scatter_proportion = reader.read_param(params, 'fixed_scatter_proportion', True) if self.fixed_scatter_proportion == True: @@ -67,6 +67,7 @@ def InternalConfigure(self, params): self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') self.path_to_missing_track_radiation_loss_data_numpy_file = '/host/' self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all4.txt') + self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '') if not os.path.exists(self.shake_spectrum_parameters_json_path): raise IOError('Shake spectrum path does not exist') @@ -137,7 +138,47 @@ def composite_gaussian(self, A_array, sigma_array): for A, sigma in zip(A_array, sigma_array): ans += self.gaussian(x_array, A, sigma, 0) return ans - + + def asym_triangle(self, x, scale1, scale2, center, exponent=1): + index_below = np.where(x=center) + f_below = 1-np.abs((x-center)/scale1)**exponent + f_above = 1-np.abs((x-center)/scale2)**exponent + f_below[np.abs(x-center)>=np.abs(scale1)]=0. + f_above[np.abs(x-center)>=np.abs(scale2)]=0. + f = np.zeros(len(x)) + f[index_above] = f_above[index_above] + f[index_below] = f_below[index_below] + return f + + def smeared_triangle(self, x, center, scale1, scale2, exponent, sigma, amplitude): + max_energy = 1000 + dx = x[1]-x[0]#E[1]-E[0] + n_dx = round(max_energy/dx) + x_smearing = np.arange(-n_dx*dx, n_dx*dx, dx) + x_triangle = np.arange(min(x)-max_energy, max(x)+max_energy, dx) + smearing = gaussian(x_smearing, 1, sigma, 0) + triangle = asym_triangle(x_triangle, scale1, scale2, center, exponent) + triangle_smeared = signal.convolve(triangle, smearing, mode='same') + triangle_smeared_norm = triangle_smeared/np.sum(triangle_smeared)*amplitude + return np.interp(x, x_triangle, triangle_smeared_norm) + + def std_smeared_triangle(self, center, scale1, scale2, exponent, sigma): + x_array = std_eV_array() + ans = self.smeared_triangle(x_array, center, scale1, scale2, exponent, sigma, 1) + return ans + + def composite_gaussian_lorentzian(self, sigma): + x_array = self.std_eV_array() + w_g = x_array/sigma + gamma = 0.8*sigma + w_l = x_array/gamma + lorentzian = 1./(gamma*np.pi)*1./(1+(w_l**2)) + gaussian = 1./(np.sqrt(2.*np.pi)*sigma)*np.exp(-0.5*w_g**2) + p = 0.8 + composite_function = p*gaussian+(1-p)*lorentzian + return composite_function + # normalizes a function, but depends on binning. # Only to be used for functions evaluated on the SELA def normalize(self, f): @@ -281,6 +322,12 @@ def convolve_composite_gaussian(self, func_to_convolve, A_array, sigma_array): ans_normed = self.normalize(ans) return ans_normed + def convolve_composite_gaussian_lorentzian(self, func_to_convolve, sigma): + resolution_f = self.composite_gaussian_lorentzian(sigma) + ans = signal.convolve(resolution_f, func_to_convolve, mode='same') + ans_normed = self.normalize(ans) + return ans_normed + def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt) x_data = ins_resolution_data.T[0] @@ -299,6 +346,12 @@ def convolve_ins_resolution(self, working_spectrum): normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum + def convolve_smeared_triangle(self, func_to_convolve, center, scale1, scale2, exponent, sigma): + resolution_f = self.std_smeared_triangle(center, scale1, scale2, exponent, sigma) + ans = signal.convolve(resolution_f, func_to_convolve, mode = 'same') + ans_normed = normalize(ans) + return ans_normed + def least_square(self, bin_centers, hist, params): # expectation expectation = self.spectrum_func_ftc(bin_centers, *params) @@ -963,3 +1016,338 @@ def fit_data_ftc_2(self, freq_bins, data_hist_freq): 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results + +def make_spectrum_smeared_triangle(self, prob_parameter, center, scale1, scale2, exponent, sigma, emitted_peak='shake'): + current_path = get_current_path() + # check_existence_of_scatter_files() + #filenames = list_files('scatter_spectra_files') + p = np.zeros(len(gases)) + p = scatter_proportion + scatter_spectra = np.load('scatter_spectra_file/scatter_spectra.npy', allow_pickle = True) + en_array = std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_smeared_triangle(current_working_spectrum, center, scale1, scale2, exponent, sigma) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += current_working_spectrum + N = len(gases) + for M in range(1, max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(sp.signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(len(gases))): + coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M + current_full_spectrum += coefficient*current_working_spectrum + + return current_full_spectrum + + def spectrum_func_smeared_triangle(self, bins_Hz, *p0): + + B_field = p0[0] + amplitude = p0[1] + prob_parameter = p0[2] + center = p0[3] + scale1 = p0[4] + scale2 = p0[5] + exponent = p0[6] + sigma = p0[7] + + x_eV = Energy(bins_Hz, B_field) + en_loss_array = std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + en_array_rev = flip_array(-1*en_loss_array) + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = kr_line*1000 - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_smeared_triangle(prob_parameter, center, scale1, scale2, exponent, sigma) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + return f + + def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_files() + bins_Hz = freq_bins + RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + bins_Hz_nonzero , data_hist_nonzero , data_hist_err = get_only_nonzero_bins(bins_Hz, data_hist_freq) + # Initial guesses for curve_fit + B_field_guess = central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + prob_parameter_guess = 0.5 + center_guess = 0 + scale_guess = 5 + exponent_guess = 1 + sigma_guess = 3 + # Bounds for curve_fit + B_field_min = central_frequency_to_B_field(bins_Hz[0]) + B_field_max = central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + center_min = -5 + center_max = 5 + width_min = 0 + width_max = Energy(bins_Hz[0], B_field_guess) - Energy(bins_Hz[-1], B_field_guess) + exponent_min = 0.5 + exponent_max = 2 + + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess, center_guess, scale_guess, scale_guess, exponent_guess, sigma_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (center_min, center_max), (width_min, width_max), (width_min, width_max), (exponent_min, exponent_max), (width_min, width_max)] + #step_size = [1e-6, 5, 500, 0.1] + # Actually do the fitting + print(p0_guess) + print(p0_bounds) + m_binned = Minuit.from_array_func(lambda p: chi2_Poisson(bins_Hz, data_hist_freq, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + prob_parameter_fit = params[2] + center_fit = params[3] + scale1_fit = params[4] + scale2_fit = params[5] + exponent_fit = params[6] + sigma_fit = params[7] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + prob_parameter_fit_err = perr[2] + center_fit_err = perr[3] + scale1_fit_err = perr[4] + scale2_fit_err = perr[5] + exponent_fit_err = perr[6] + sigma_fit_err = perr[7] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = spectrum_func(bins_Hz,*params) + fit_keV = flip_array(fit_Hz) + bins_keV = Energy(bins_Hz, B_field_fit)/1000 + bins_keV = flip_array(bins_keV) + reduced_chi2 = reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = len(params)) + if print_params == True: + output_string = 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit) + ' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' + output_string += '-----------------\n' + output_string += 'Center = {:.4e}'.format(center_fit) + ' +/- {:.4e}'.format(center_fit_err) + '\n' + output_string += '-----------------\n' + output_string += 'Scale1 = {:.4e}'.format(scale1_fit) + ' +/- {:.4e}'.format(scale1_fit_err) + '\n' + output_string += '-----------------\n' + output_string += 'Scale2 = {:.4e}'.format(scale2_fit) + ' +/- {:.4e}'.format(scale2_fit_err) + '\n' + output_string += '-----------------\n' + output_string += 'Exponent = {:.4e}'.format(exponent_fit) + ' +/- {:.4e}'.format(exponent_fit_err) + '\n' + output_string += '-----------------\n' + output_string += 'Sigma = {:.4e}'.format(sigma_fit) + ' +/- {:.4e}'.format(sigma_fit_err) + '\n' + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'prob_parameter_fit': prob_parameter_fit, + 'prob_parameter_fit_err': prob_parameter_fit_err, + 'center_fit': scatter_proportion_fit, + 'center_fit_err': scatter_proportion_fit_err, + 'scale1_fit': scale1_fit, + 'scale1_fit_err': scale1_fit_err, + 'scale2_fit': scale2_fit, + 'scale2_fit_err': scale2_fit_err, + 'exponent_fit': exponent_fit, + 'exponent_fit_err': exponent_fit_err, + 'sigma_fit': sigma_fit, + 'sigma_fit_err': sigma_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results + + def make_spectrum_composite_gaussian_lorentzian(self, prob_parameter, sigma, emitted_peak='shake'): + p = scatter_proportion + scatter_spectra = np.load('scatter_spectra_file/scatter_spectra.npy', allow_pickle = True) + en_array = std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = shake_spectrum() + current_working_spectrum = convolve_composite_gaussian_lorentzian(current_working_spectrum, sigma) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += 2*zeroth_order_peak + N = len(gases) + for M in range(1, max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = normalize(sp.signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(len(gases))): + coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M + current_full_spectrum += coefficient*current_working_spectrum + return current_full_spectrum + + def spectrum_func(self, bins_Hz, eff_array, *p0): + + B_field = p0[0] + amplitude = p0[1] + prob_parameter = p0[2] + sigma = p0[3] + + x_eV = Energy(bins_Hz, B_field) + en_loss_array = std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + en_array_rev = flip_array(-1*en_loss_array) + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = kr_line*1000 - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = make_spectrum(prob_parameter, sigma) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def fit_data(RF_ROI_MIN, freq_bins, data_hist_freq, print_params=True): + t = time.time() + check_existence_of_scatter_files() + bins_Hz = freq_bins + RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + + bins_Hz_nonzero , data_hist_nonzero , data_hist_err = get_only_nonzero_bins(bins_Hz, data_hist_freq) + # Initial guesses for curve_fit + B_field_guess = central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + FWHM_eV_guess = 5 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + sigma_guess = 5 + gamma_guess = 3 + gaussian_portion_guess = 0.5 + # Bounds for curve_fit + B_field_min = central_frequency_to_B_field(bins_Hz[0]) + B_field_max = central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + FWHM_eV_min = 0 + FWHM_eV_max = Energy(bins_Hz[0], B_field_guess) + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + mu_min = -FWHM_eV_max + mu_max = FWHM_eV_max + gaussian_portion_min = 1e-5 + gaussian_portion_max = 1 + N = len(gases) + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess, sigma_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (FWHM_eV_min, FWHM_eV_max)] + # Actually do the fitting + print(p0_guess) + print(p0_bounds) + m_binned = Minuit.from_array_func(lambda p: chi2_Poisson(bins_Hz, data_hist_freq, eff_array, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + prob_parameter_fit = params[2] + sigma_fit = params[3] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + prob_parameter_fit_err = perr[2] + sigma_fit_err = perr[3] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = spectrum_func(bins_Hz, eff_array, *params) + fit_keV = flip_array(fit_Hz) + bins_keV = Energy(bins_Hz, B_field_fit)/1000 + bins_keV = flip_array(bins_keV) + reduced_chi2 = reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = len(params)) + + if print_params == True: + output_string = 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)\ + +' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' + output_string += '-----------------\n' + output_string += 'sigma = {:.2e}'.format(sigma_fit) + ' +/- {:.4e}\n'.format(sigma_fit_err) + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'prob_parameter_fit': prob_parameter_fit, + 'prob_parameter_fit_err': prob_parameter_fit_err, + 'sigma_fit': sigma_fit, + 'sigma_fit_err': sigma_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } From d0bf786e690452d46b44b0d66e17465536284835 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Thu, 22 Oct 2020 17:37:27 -0400 Subject: [PATCH 043/199] Enabled FakeDataGenerator to use complex ls as func of K --- mermithid/misc/FakeTritiumDataFunctions.py | 7 ++++--- .../processors/TritiumSpectrum/FakeDataGenerator.py | 10 ++++++++++ mermithid/processors/misc/MultiGasComplexLineShape.py | 1 + 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 88364d5e..dccfee3b 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -20,6 +20,7 @@ from mermithid.misc.ConversionFunctions import * import matplotlib.pyplot as plt +import types """ Constants and functions used by processors/TritiumSpectrum/FakeDataGenerator.py @@ -264,10 +265,10 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - lineshape_rates = complexLineShape.spectrum_func_ftc(K_lineshape/1000., B_field, 1, ls_params[1]) + lineshape_rates = complexLineShape.make_spectrum_ftc(ls_params[1], emitted_peak='dirac') #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000.,ls_params[0], 0, 1, ls_params[1]) fig = plt.figure() - plt.plot(K_lineshape/1000., lineshape_rates) + plt.plot(K_lineshape, lineshape_rates) plt.xlabel('Energy shift (eV)') plt.ylabel('Complex lineshape rate') plt.savefig('complex_lineshape_rates.pdf') @@ -308,7 +309,7 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - lineshape_rates = complexLineShape.spectrum_func_ftc(K_lineshape/1000., B_field, 1, ls_params[1]) + lineshape_rates = complexLineShape.make_spectrum_ftc(ls_params[1], emitted_peak='dirac') #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., ls_params[0], 0, 1, ls_params[1]) bkgd_rates = np.full(len(K), bkgd_rate()) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 8126e7fe..55d222c9 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -290,6 +290,16 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 #Options of kinetic energies to be sampled self.Koptions = np.arange(Kmin_eff, Kmax_eff, step_size) + def K_ls_complex(): + dE = self.Koptions[1] - self.Koptions[0] + energy_half_range = max(max_energy, abs(min_energy)) + n_dE_pos = round(energy_half_range/dE) #Number of steps for the lineshape for energies > 0 + n_dE_neg = round(energy_half_range/dE) #Same, for energies < 0 + K_lineshape = np.arange(-n_dE_neg*dE, n_dE_pos*dE, dE) + return K_lineshape + + self.complexLineShape.std_eV_array = K_ls_complex + if efficiency_dict is not None: logger.info('Evaluating efficiencies') efficiency_mean, efficiency_error = efficiency_from_interpolation(self.Koptions, efficiency_dict, B_field) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 4b7d170c..e89d679f 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -811,6 +811,7 @@ def spectrum_func_ftc(self, bins_Hz, *p0): nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] full_spectrum = self.make_spectrum_ftc(prob_parameter) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum) f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) return f From 6d99c11a3ab0c327e3889c89788aa2ce15a58982 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Thu, 22 Oct 2020 19:34:29 -0400 Subject: [PATCH 044/199] Fixed scatter peak bug; removed temporary plots/printing --- mermithid/misc/FakeTritiumDataFunctions.py | 17 --------- .../TritiumSpectrum/FakeDataGenerator.py | 3 -- .../misc/MultiGasComplexLineShape.py | 35 +++++-------------- 3 files changed, 8 insertions(+), 47 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index dccfee3b..13436f82 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -19,9 +19,6 @@ from mermithid.misc.Constants import * from mermithid.misc.ConversionFunctions import * -import matplotlib.pyplot as plt -import types - """ Constants and functions used by processors/TritiumSpectrum/FakeDataGenerator.py """ @@ -266,13 +263,6 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': lineshape_rates = complexLineShape.make_spectrum_ftc(ls_params[1], emitted_peak='dirac') - #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000.,ls_params[0], 0, 1, ls_params[1]) - fig = plt.figure() - plt.plot(K_lineshape, lineshape_rates) - plt.xlabel('Energy shift (eV)') - plt.ylabel('Complex lineshape rate') - plt.savefig('complex_lineshape_rates.pdf') - plt.show() beta_rates = np.zeros(len(K)) for i,ke in enumerate(K): @@ -280,12 +270,6 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, #Convolving convolved = convolve(beta_rates, lineshape_rates, mode='same') - fig = plt.figure() - plt.plot(K, convolved) - plt.xlabel('Energy (eV)') - plt.ylabel('Signal rate') - plt.savefig('spectrum_signal.pdf') - plt.show() below_Kmin = np.where(K < Kmin) np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) return convolved @@ -310,7 +294,6 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': lineshape_rates = complexLineShape.make_spectrum_ftc(ls_params[1], emitted_peak='dirac') - #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., ls_params[0], 0, 1, ls_params[1]) bkgd_rates = np.full(len(K), bkgd_rate()) if len(K) < len(K_lineshape): diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 55d222c9..f5743f2b 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -349,9 +349,6 @@ def K_ls_complex(): ratesS[ratesS<0.] = 0. ratesB[ratesB<0.] = 0. - print(ratesS) - print(len(ratesS)) - print(ratesS[2000:2010]) rate_sumS, rate_sumB = np.sum(ratesS), np.sum(ratesB) probsS = np.array(ratesS)/rate_sumS probsB = np.array(ratesB)/rate_sumB diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index e89d679f..54c30234 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -34,7 +34,6 @@ from morpho.utilities import morphologging, reader from morpho.processors import BaseProcessor from mermithid.misc import Constants, ComplexLineShapeUtilities, ConversionFunctions -import matplotlib.pyplot as plt logger = morphologging.getLogger(__name__) @@ -340,31 +339,13 @@ def combine_four_trap_resolution_from_txt(self, trap_weights): def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_array): x_data, y_data_combined, y_err_data_combined = self.combine_four_trap_resolution_from_txt(weight_array) - fig = plt.figure() - plt.errorbar(x_data, y_data_combined, yerr=y_err_data_combined) - plt.xlabel('Energy shift (eV)') - plt.ylabel('Probability (post-combining traps)') - plt.savefig('combined_trap_ins_res.pdf') - plt.show() if self.sample_ins_resolution_errors: y_data_combined = np.random.normal(y_data_combined, y_err_data_combined) - fig = plt.figure() - plt.plot(x_data, y_data_combined) - plt.xlabel('Energy shift (eV)') - plt.ylabel('Probability (post-sampling)') - plt.savefig('sampled_combined_trap_ins_res.pdf') - plt.show() f = interpolate.interp1d(x_data, y_data_combined) x_array = self.std_eV_array() y_array = np.zeros(len(x_array)) index_within_range_of_xdata = np.where((x_array >= x_data[0]) & (x_array <= x_data[-1])) y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]) - fig = plt.figure() - plt.plot(x_array, y_array) - plt.xlabel('Energy shift (eV)') - plt.ylabel('Probability (post-interpolation)') - plt.savefig('interpolated_combined_trap_ins_res.pdf') - plt.show() convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum @@ -457,8 +438,8 @@ def make_spectrum(self, gauss_FWHM_eV, prob_parameter, scatter_proportion, emitt current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) coefficient = factorial(sum(combination)) for component, i in zip(combination, range(len(self.gases))): - coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M - current_full_spectrum += coefficient*current_working_spectrum + coefficient = coefficient/factorial(component)*p[i]**component + current_full_spectrum += coefficient*current_working_spectrum*prob_parameter**M return current_full_spectrum # Produces a spectrum in real energy that can now be evaluated off of the SELA. @@ -629,8 +610,8 @@ def make_spectrum_1(self, gauss_FWHM_eV, prob_parameter, emitted_peak='shake'): current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) coefficient = factorial(sum(combination)) for component, i in zip(combination, range(len(self.gases))): - coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M - current_full_spectrum += coefficient*current_working_spectrum + coefficient = coefficient/factorial(component)*p[i]**component + current_full_spectrum += coefficient*current_working_spectrum*prob_parameter**M return current_full_spectrum def spectrum_func_1(self, bins_Hz, *p0): @@ -789,8 +770,8 @@ def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) coefficient = factorial(sum(combination)) for component, i in zip(combination, range(len(self.gases))): - coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M - current_full_spectrum += coefficient*current_working_spectrum + coefficient = coefficient/factorial(component)*p[i]**component + current_full_spectrum += coefficient*current_working_spectrum*prob_parameter**M return current_full_spectrum def spectrum_func_ftc(self, bins_Hz, *p0): @@ -935,8 +916,8 @@ def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak=' current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) coefficient = factorial(sum(combination)) for component, i in zip(combination, range(len(self.gases))): - coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M - current_full_spectrum += coefficient*current_working_spectrum + coefficient = coefficient/factorial(component)*p[i]**component + current_full_spectrum += coefficient*current_working_spectrum*prob_parameter**M return current_full_spectrum def spectrum_func_ftc_2(self, bins_Hz, *p0): From dc418e72abf6dc720708d4515b5992cca496ba4d Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Sat, 24 Oct 2020 06:33:38 -0400 Subject: [PATCH 045/199] Set bin width of std_eV_array manually --- mermithid/misc/FakeTritiumDataFunctions.py | 32 ++++++++++++++++++- .../TritiumSpectrum/FakeDataGenerator.py | 14 +++++--- .../processors/misc/KrComplexLineShape.py | 4 +-- .../misc/MultiGasComplexLineShape.py | 4 +-- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 13436f82..98b46b0f 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -19,6 +19,8 @@ from mermithid.misc.Constants import * from mermithid.misc.ConversionFunctions import * +import matplotlib.pyplot as plt + """ Constants and functions used by processors/TritiumSpectrum/FakeDataGenerator.py """ @@ -262,7 +264,20 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': + #bins_Hz = Frequency(K_lineshape, B_field) + #lineshape_rates = complexLineShape.spectrum_func_ftc(bins_Hz, B_field, 1, ls_params[1]) + #lineshape_rates = complexLineShape.make_spectrum_1(ls_params[0]*2*np.sqrt(2*np.log(2)), ls_params[1], emitted_peak='dirac') lineshape_rates = complexLineShape.make_spectrum_ftc(ls_params[1], emitted_peak='dirac') + #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., 0., 1., ls_params[1]) + lineshape_rates = np.flipud(lineshape_rates) + + + fig = plt.figure() + plt.plot(K_lineshape, lineshape_rates) + plt.xlabel('Energy shift (eV)') + plt.ylabel('Complex lineshape rate') + plt.savefig('complex_lineshape_rates.pdf') + beta_rates = np.zeros(len(K)) for i,ke in enumerate(K): @@ -270,6 +285,15 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, #Convolving convolved = convolve(beta_rates, lineshape_rates, mode='same') + + + fig = plt.figure() + plt.plot(K, convolved) + plt.xlabel('Energy (eV)') + plt.ylabel('Signal rate') + plt.savefig('spectrum_signal.pdf') + + below_Kmin = np.where(K < Kmin) np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) return convolved @@ -293,8 +317,13 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': + #bins_Hz = Frequency(K_lineshape, B_field) + #lineshape_rates = complexLineShape.spectrum_func_ftc(bins_Hz, B_field, 1, ls_params[1]) + #lineshape_rates = complexLineShape.make_spectrum_1(ls_params[0]*2*np.sqrt(2*np.log(2)), ls_params[1], emitted_peak='dirac') lineshape_rates = complexLineShape.make_spectrum_ftc(ls_params[1], emitted_peak='dirac') - + #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., 0., 1., ls_params[1]) + lineshape_rates = np.flipud(lineshape_rates) + bkgd_rates = np.full(len(K), bkgd_rate()) if len(K) < len(K_lineshape): raise Exception("lineshape array is longer than Koptions") @@ -303,6 +332,7 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, convolved = convolve(bkgd_rates, lineshape_rates, mode='same') below_Kmin = np.where(K < Kmin) np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) + return convolved diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index f5743f2b..47aea896 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -17,6 +17,7 @@ from morpho.processors import BaseProcessor from mermithid.misc.FakeTritiumDataFunctions import * from mermithid.processors.misc.MultiGasComplexLineShape import MultiGasComplexLineShape +#from mermithid.processors.misc.KrComplexLineShape import KrComplexLineShape from mermithid.misc import Constants, ComplexLineShapeUtilities, ConversionFunctions logger = morphologging.getLogger(__name__) @@ -147,22 +148,23 @@ def InternalConfigure(self, params): # Setup and configure lineshape processor complexLineShape_config = { - 'gases': ["H2","He"], + 'gases': ["H2", "He"], 'max_scatters': self.NScatters, 'fixed_scatter_proportion': True, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below 'gas_scatter_proportion': [self.scatter_proportion, 1.-self.scatter_proportion], + #'gas1_scatter_proportion': self.scatter_proportion, #, 1.-self.scatter_proportion], 'use_simulated_inst_reso': True, 'use_combined_four_trap_inst_reso': False, # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to # be increased for larger datasets. - 'num_points_in_std_array': 10000, + 'num_points_in_std_array': 35838, 'base_shape': 'dirac', 'sample_ins_resolution_errors': True, 'use_combined_four_trap_inst_reso': True, 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path, - 'path_to_scatter_spectra_file':self.detailed_scatter_spectra_path + 'path_to_scatter_spectra_file':self.detailed_scatter_spectra_path, } logger.info('Setting up complex lineshape object') self.complexLineShape = MultiGasComplexLineShape("complexLineShape") @@ -273,10 +275,10 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 nstdevs = 7 #Number of standard deviations (of size broadening) below Kmin and above Q-m to generate data, for the gaussian case FWHM_convert = 2*math.sqrt(2*math.log(2)) if lineshape=='gaussian': - max_energy = nstdevs*params[0] + max_energy = 1000 #nstdevs*params[0] min_energy = -1000 elif lineshape=='simplified_scattering' or lineshape=='simplified' or lineshape=='detailed_scattering' or lineshape=='detailed': - max_energy = nstdevs/FWHM_convert*params[0] + max_energy = 1000 #nstdevs/FWHM_convert*params[0] min_energy = -1000 Kmax_eff = Kmax+max_energy #Maximum energy for data is slightly above Kmax>Q-m @@ -290,6 +292,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 #Options of kinetic energies to be sampled self.Koptions = np.arange(Kmin_eff, Kmax_eff, step_size) + """ def K_ls_complex(): dE = self.Koptions[1] - self.Koptions[0] energy_half_range = max(max_energy, abs(min_energy)) @@ -299,6 +302,7 @@ def K_ls_complex(): return K_lineshape self.complexLineShape.std_eV_array = K_ls_complex + """ if efficiency_dict is not None: logger.info('Evaluating efficiencies') diff --git a/mermithid/processors/misc/KrComplexLineShape.py b/mermithid/processors/misc/KrComplexLineShape.py index 980be3c8..0e39e416 100644 --- a/mermithid/processors/misc/KrComplexLineShape.py +++ b/mermithid/processors/misc/KrComplexLineShape.py @@ -54,7 +54,7 @@ def InternalConfigure(self, params): self.fix_scatter_proportion = reader.read_param(params, 'fix_scatter_proportion', True) if self.fix_scatter_proportion == True: self.scatter_proportion = reader.read_param(params, 'gas1_scatter_proportion', 0.8) - logger.info('Using an H2 scatter proportion of {} with gases {}'.format(self.gases, self.scatter_proportion)) + logger.info('Using an H2 scatter proportion of {} with gases {}'.format(self.scatter_proportion, self.gases)) # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown self.num_points_in_std_array = reader.read_param(params, 'num_points_in_std_array', 10000) @@ -63,7 +63,7 @@ def InternalConfigure(self, params): self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') self.base_shape = reader.read_param(params, 'base_shape', 'shake') self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') - self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all.txt') + self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max15.5_alltraps.txt') if self.base_shape=='shake' and not os.path.exists(self.shake_spectrum_parameters_json_path): raise IOError('Shake spectrum path does not exist') diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 54c30234..f73774b9 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -368,12 +368,12 @@ def chi2_Poisson(self, bin_centers, data_hist_freq, params): zero_bins_index = np.where(data_hist_freq == 0) # expectation if self.fixed_scatter_proportion: - if self.fit_ftc: + if self.use_simulated_inst_reso: fit_Hz = self.spectrum_func_ftc(bin_centers, *params) else: fit_Hz = self.spectrum_func_1(bin_centers, *params) else: - if self.fit_ftc: + if self.use_simulated_inst_reso: fit_Hz = self.spectrum_func_ftc_2(bin_centers, *params) else: fit_Hz = self.spectrum_func(bin_centers, *params) From aed8ddc1bff091ea7f54bd8bef54d7fe0a05e7aa Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Sat, 24 Oct 2020 13:10:15 -0400 Subject: [PATCH 046/199] Changed default simulated res to T2 version --- mermithid/processors/misc/MultiGasComplexLineShape.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index f73774b9..d6ccbb18 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -68,9 +68,9 @@ def InternalConfigure(self, params): self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') self.path_to_missing_track_radiation_loss_data_numpy_file = reader.read_param(params, 'rad_loss_path', '/termite/analysis_input/complex-lineshape-inputs') - self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/res_all_conversion_max15.5_alltraps.txt') + self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_all.txt') self.use_combined_four_trap_inst_reso = reader.read_param(params, 'use_combined_four_trap_inst_reso', False) - self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max15.5_trap1.txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max15.5_trap2.txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max15.5_trap3.txt', '/termite/analysis_input/complex-lineshape-inputs/res_all_conversion_max15.5_trap4.txt']) + self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/termite/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) if not os.path.exists(self.shake_spectrum_parameters_json_path) and self.base_shape=='shake': raise IOError('Shake spectrum path does not exist') From 6cf517972963f0dfbc20a5671ba8d436be95ca46 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Sun, 25 Oct 2020 13:47:29 -0400 Subject: [PATCH 047/199] push the changes in there --- mermithid/processors/misc/MultiGasComplexLineShape.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 02163860..cffa2e88 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -55,7 +55,8 @@ def InternalConfigure(self, params): self.fixed_scatter_proportion = reader.read_param(params, 'fixed_scatter_proportion', True) if self.fixed_scatter_proportion == True: self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) - self.fit_ftc = reader.read_param(params, 'fit_ftc', True) + self.use_simulated_inst_reso = reader.read_param(params, 'use_simulated_inst_reso', True) + self.use_simulated_four_trap_simulated_inst_reso_combined = reader.read_param(params, 'use_four_trap_simulated_inst_reso_combined', True) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', False) # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown @@ -93,12 +94,12 @@ def InternalRun(self): # kr17kev_in_hz = guess*(bins[1]-bins[0])+bins[0] #self.B_field = B(17.8, kr17kev_in_hz + 0) if self.fixed_scatter_proportion == True: - if self.fit_ftc == True: + if self.use_simulated_inst_reso == True: self.results = self.fit_data_ftc(freq_bins, data_hist_freq) else: self.results = self.fit_data_1(freq_bins, data_hist_freq) else: - if self.fit_ftc == True: + if self.use_simulated_inst_reso == True: self.results = self.fit_data_ftc_2(freq_bins, data_hist_freq) else: self.results = self.fit_data(freq_bins, data_hist_freq) From 2cd91fa6abb17b6b0291cd89ea69774b4ae764ed Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 2 Nov 2020 18:16:46 -0500 Subject: [PATCH 048/199] add fitting with composite gaussian lorentzian resolution --- .../misc/MultiGasComplexLineShape.py | 188 ++++++++++-------- test_analysis/Complex_line_shape_fitter.py | 9 +- 2 files changed, 105 insertions(+), 92 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index cffa2e88..b7b3b070 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -55,9 +55,9 @@ def InternalConfigure(self, params): self.fixed_scatter_proportion = reader.read_param(params, 'fixed_scatter_proportion', True) if self.fixed_scatter_proportion == True: self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) - self.use_simulated_inst_reso = reader.read_param(params, 'use_simulated_inst_reso', True) - self.use_simulated_four_trap_simulated_inst_reso_combined = reader.read_param(params, 'use_four_trap_simulated_inst_reso_combined', True) - self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', False) + self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) + # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution + self.resolution_function = reader.read_param(params, 'resolution_function', '') # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown self.num_points_in_std_array = reader.read_param(params, 'num_points_in_std_array', 10000) @@ -68,7 +68,7 @@ def InternalConfigure(self, params): self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') self.path_to_missing_track_radiation_loss_data_numpy_file = '/host/' self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all4.txt') - self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '') + self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') if not os.path.exists(self.shake_spectrum_parameters_json_path): raise IOError('Shake spectrum path does not exist') @@ -94,14 +94,16 @@ def InternalRun(self): # kr17kev_in_hz = guess*(bins[1]-bins[0])+bins[0] #self.B_field = B(17.8, kr17kev_in_hz + 0) if self.fixed_scatter_proportion == True: - if self.use_simulated_inst_reso == True: + if self.resolution_function == 'simulated_resolution': self.results = self.fit_data_ftc(freq_bins, data_hist_freq) - else: + elif self.resolution_function == 'gaussian_resolution': self.results = self.fit_data_1(freq_bins, data_hist_freq) + elif self.resolution_function == 'gaussian_lorentzian_composite_resolution': + self.results = self.fit_data_composite_gaussian_lorentzian(freq_bins, data_hist_freq) else: - if self.use_simulated_inst_reso == True: + if self.resolution_function == 'simulated_resolution': self.results = self.fit_data_ftc_2(freq_bins, data_hist_freq) - else: + elif self.resolution_function == 'gaussian_resolution': self.results = self.fit_data(freq_bins, data_hist_freq) return True @@ -384,6 +386,15 @@ def chi2_Poisson(self, bin_centers, data_hist_freq, params): chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() return chi2 + def chi_2_Poisson_composite_gaussian_lorentzian_reso(self, bin_centers, data_hist_freq, eff_array, params): + nonzero_bins_index = np.where(data_hist_freq != 0) + zero_bins_index = np.where(data_hist_freq == 0) + # expectation + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian(bin_centers, eff_array, *params) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + return chi2 + def reduced_chi2_Pearson_Neyman_composite(self, data_hist_freq, fit_Hz, number_of_parameters): nonzero_bins_index = np.where(data_hist_freq != 0)[0] zero_bins_index = np.where(data_hist_freq == 0)[0] @@ -1018,39 +1029,39 @@ def fit_data_ftc_2(self, freq_bins, data_hist_freq): } return dictionary_of_fit_results -def make_spectrum_smeared_triangle(self, prob_parameter, center, scale1, scale2, exponent, sigma, emitted_peak='shake'): - current_path = get_current_path() - # check_existence_of_scatter_files() - #filenames = list_files('scatter_spectra_files') - p = np.zeros(len(gases)) - p = scatter_proportion - scatter_spectra = np.load('scatter_spectra_file/scatter_spectra.npy', allow_pickle = True) - en_array = std_eV_array() - current_full_spectrum = np.zeros(len(en_array)) - if emitted_peak == 'lorentzian': - current_working_spectrum = self.std_lorenztian_17keV() - elif emitted_peak == 'shake': - current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() - current_working_spectrum = self.convolve_smeared_triangle(current_working_spectrum, center, scale1, scale2, exponent, sigma) - zeroth_order_peak = current_working_spectrum - current_full_spectrum += current_working_spectrum - N = len(gases) - for M in range(1, max_scatters + 1): - gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) - for combination in gas_scatter_combinations: - #print(combination) - entry_str = '' - for component, gas_type in zip(combination, gases): - entry_str += gas_type - entry_str += str(component).zfill(2) - current_working_spectrum = scatter_spectra.item()[entry_str] - current_working_spectrum = self.normalize(sp.signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) - coefficient = factorial(sum(combination)) - for component, i in zip(combination, range(len(gases))): - coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M - current_full_spectrum += coefficient*current_working_spectrum - - return current_full_spectrum + def make_spectrum_smeared_triangle(self, prob_parameter, center, scale1, scale2, exponent, sigma, emitted_peak='shake'): + current_path = get_current_path() + # check_existence_of_scatter_files() + #filenames = list_files('scatter_spectra_files') + p = np.zeros(len(gases)) + p = scatter_proportion + scatter_spectra = np.load('scatter_spectra_file/scatter_spectra.npy', allow_pickle = True) + en_array = std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_smeared_triangle(current_working_spectrum, center, scale1, scale2, exponent, sigma) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += current_working_spectrum + N = len(gases) + for M in range(1, max_scatters + 1): + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(sp.signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(len(gases))): + coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M + current_full_spectrum += coefficient*current_working_spectrum + + return current_full_spectrum def spectrum_func_smeared_triangle(self, bins_Hz, *p0): @@ -1197,74 +1208,75 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print } return dictionary_of_fit_results - def make_spectrum_composite_gaussian_lorentzian(self, prob_parameter, sigma, emitted_peak='shake'): - p = scatter_proportion - scatter_spectra = np.load('scatter_spectra_file/scatter_spectra.npy', allow_pickle = True) - en_array = std_eV_array() + def make_spectrum_composite_gaussian_lorentzian(self, survival_prob, sigma, emitted_peak='shake'): + p = self.scatter_proportion + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() current_full_spectrum = np.zeros(len(en_array)) if emitted_peak == 'lorentzian': - current_working_spectrum = std_lorenztian_17keV() + current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': - current_working_spectrum = shake_spectrum() - current_working_spectrum = convolve_composite_gaussian_lorentzian(current_working_spectrum, sigma) + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_composite_gaussian_lorentzian(current_working_spectrum, sigma) zeroth_order_peak = current_working_spectrum - current_full_spectrum += 2*zeroth_order_peak - N = len(gases) - for M in range(1, max_scatters + 1): + current_full_spectrum += zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + relative_reconstruction_eff = np.exp(-0.643*M**0.74) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) entry_str = '' - for component, gas_type in zip(combination, gases): + for component, gas_type in zip(combination, self.gases): entry_str += gas_type entry_str += str(component).zfill(2) current_working_spectrum = scatter_spectra.item()[entry_str] - current_working_spectrum = normalize(sp.signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) coefficient = factorial(sum(combination)) - for component, i in zip(combination, range(len(gases))): - coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M - current_full_spectrum += coefficient*current_working_spectrum + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + for i in range(0, M+1): + coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum - def spectrum_func(self, bins_Hz, eff_array, *p0): + def spectrum_func_composite_gaussian_lorentzian(self, bins_Hz, eff_array, *p0): B_field = p0[0] amplitude = p0[1] - prob_parameter = p0[2] + survival_prob = p0[2] sigma = p0[3] - x_eV = Energy(bins_Hz, B_field) - en_loss_array = std_eV_array() + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() en_loss_array_min = en_loss_array[0] en_loss_array_max = en_loss_array[len(en_loss_array)-1] - en_array_rev = flip_array(-1*en_loss_array) f = np.zeros(len(x_eV)) f_intermediate = np.zeros(len(x_eV)) - x_eV_minus_line = kr_line*1000 - x_eV + x_eV_minus_line = Constants.kr_k_line_e() - x_eV zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - full_spectrum = make_spectrum(prob_parameter, sigma) + full_spectrum = self.make_spectrum_composite_gaussian_lorentzian(survival_prob, sigma) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f_intermediate = f_intermediate*eff_array f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) return f - def fit_data(RF_ROI_MIN, freq_bins, data_hist_freq, print_params=True): + def fit_data_composite_gaussian_lorentzian(self, freq_bins, data_hist_freq, print_params=True): t = time.time() - check_existence_of_scatter_files() - bins_Hz = freq_bins + RF_ROI_MIN + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) - - bins_Hz_nonzero , data_hist_nonzero , data_hist_err = get_only_nonzero_bins(bins_Hz, data_hist_freq) # Initial guesses for curve_fit - B_field_guess = central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) amplitude_guess = np.sum(data_hist_freq)/2 FWHM_eV_guess = 5 prob_parameter_guess = 0.5 @@ -1273,12 +1285,12 @@ def fit_data(RF_ROI_MIN, freq_bins, data_hist_freq, print_params=True): gamma_guess = 3 gaussian_portion_guess = 0.5 # Bounds for curve_fit - B_field_min = central_frequency_to_B_field(bins_Hz[0]) - B_field_max = central_frequency_to_B_field(bins_Hz[-1]) + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) amplitude_min = 1e-5 amplitude_max = np.sum(data_hist_freq)*3 FWHM_eV_min = 0 - FWHM_eV_max = Energy(bins_Hz[0], B_field_guess) + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) prob_parameter_min = 1e-5 prob_parameter_max = 1 scatter_proportion_min = 1e-5 @@ -1287,13 +1299,11 @@ def fit_data(RF_ROI_MIN, freq_bins, data_hist_freq, print_params=True): mu_max = FWHM_eV_max gaussian_portion_min = 1e-5 gaussian_portion_max = 1 - N = len(gases) + N = len(self.gases) p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess, sigma_guess] p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (FWHM_eV_min, FWHM_eV_max)] # Actually do the fitting - print(p0_guess) - print(p0_bounds) - m_binned = Minuit.from_array_func(lambda p: chi2_Poisson(bins_Hz, data_hist_freq, eff_array, p), + m_binned = Minuit.from_array_func(lambda p: self.chi_2_Poisson_composite_gaussian_lorentzian_reso(bins_Hz, data_hist_freq, eff_array, p), start = p0_guess, limit = p0_bounds, throw_nan = True @@ -1303,32 +1313,33 @@ def fit_data(RF_ROI_MIN, freq_bins, data_hist_freq, print_params=True): B_field_fit = params[0] #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) amplitude_fit = params[1] - prob_parameter_fit = params[2] + survival_prob_fit = params[2] sigma_fit = params[3] total_counts_fit = amplitude_fit perr = m_binned.np_errors() B_field_fit_err = perr[0] amplitude_fit_err = perr[1] - prob_parameter_fit_err = perr[2] + survival_prob_fit_err = perr[2] sigma_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - fit_Hz = spectrum_func(bins_Hz, eff_array, *params) - fit_keV = flip_array(fit_Hz) - bins_keV = Energy(bins_Hz, B_field_fit)/1000 - bins_keV = flip_array(bins_keV) - reduced_chi2 = reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = len(params)) + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian(bins_Hz, eff_array, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) if print_params == True: - output_string = 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) output_string += '-----------------\n' output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) output_string += '-----------------\n' output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' output_string += '-----------------\n' - output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)\ - +' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' + output_string += 'Survival probability \n= ' + "{:.2e}".format(survival_prob_fit)\ + +' +/- ' + "{:.2e}".format(survival_prob_fit_err)+'\n' output_string += '-----------------\n' output_string += 'sigma = {:.2e}'.format(sigma_fit) + ' +/- {:.4e}\n'.format(sigma_fit_err) output_string += '-----------------\n' @@ -1343,8 +1354,8 @@ def fit_data(RF_ROI_MIN, freq_bins, data_hist_freq, print_params=True): 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, - 'prob_parameter_fit': prob_parameter_fit, - 'prob_parameter_fit_err': prob_parameter_fit_err, + 'survival_prob_fit': survival_prob_fit, + 'survival_prob_fit_err': survival_prob_fit_err, 'sigma_fit': sigma_fit, 'sigma_fit_err': sigma_fit_err, 'amplitude_fit': amplitude_fit, @@ -1352,3 +1363,4 @@ def fit_data(RF_ROI_MIN, freq_bins, data_hist_freq, print_params=True): 'data_hist_freq': data_hist_freq, 'reduced_chi2': reduced_chi2 } + return dictionary_of_fit_results diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index fe178066..e8437dc9 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -31,10 +31,11 @@ def test_complex_lineshape(self): 'bins_choice': np.linspace(0e6, 100e6, 1000), 'gases': ["H2", "Kr"], 'max_scatters': 20, - 'fixed_scatter_proportion': False, - 'fit_ftc':True, # use gaussian instrumental resolution + 'fixed_scatter_proportion': True, + # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution + 'resolution_function': 'gaussian_lorentzian_composite_resolution', # When fix_scatter_proportion is True, set the scatter proportion for gas1 below - 'gas_scatter_proportion': [0.61, 0.34], + 'gas_scatter_proportion': [0.61, 0.39], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -86,7 +87,7 @@ def test_complex_lineshape(self): plot_title += '\n with fixed scatter proportion \n {}'.format(str_gas_scatter_proportion) plt.title(plot_title) plt.tight_layout() - plt.savefig('/host/plots/fit_FTC_march_with_{}_gas_scattering_max25_fitting_H2_He.png'.format(len(complexLineShape_config['gases']))) + plt.savefig('/host/plots/fit_FTC_march_with_composite_gaussian_lorentzian_resolution.png'.format(len(complexLineShape_config['gases']))) if __name__ == '__main__': From 350f496cd79a880e3f389e4ad7e0f217cce5fa2c Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 2 Nov 2020 21:46:02 -0500 Subject: [PATCH 049/199] fix survival probability --- .../misc/MultiGasComplexLineShape.py | 26 +++++++------------ test_analysis/Complex_line_shape_fitter.py | 3 ++- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index b7b3b070..827554a7 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -55,6 +55,7 @@ def InternalConfigure(self, params): self.fixed_scatter_proportion = reader.read_param(params, 'fixed_scatter_proportion', True) if self.fixed_scatter_proportion == True: self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) + self.survival_prob = reader.read_param(params, 'survival_prob', 1) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution self.resolution_function = reader.read_param(params, 'resolution_function', '') @@ -1208,8 +1209,9 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print } return dictionary_of_fit_results - def make_spectrum_composite_gaussian_lorentzian(self, survival_prob, sigma, emitted_peak='shake'): + def make_spectrum_composite_gaussian_lorentzian(self, sigma, emitted_peak='shake'): p = self.scatter_proportion + survival_prob = self.survival_prob scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) en_array = self.std_eV_array() @@ -1244,9 +1246,8 @@ def make_spectrum_composite_gaussian_lorentzian(self, survival_prob, sigma, emit def spectrum_func_composite_gaussian_lorentzian(self, bins_Hz, eff_array, *p0): B_field = p0[0] - amplitude = p0[1] - survival_prob = p0[2] - sigma = p0[3] + amplitude = p0[1] + sigma = p0[2] x_eV = ConversionFunctions.Energy(bins_Hz, B_field) en_loss_array = self.std_eV_array() @@ -1259,7 +1260,7 @@ def spectrum_func_composite_gaussian_lorentzian(self, bins_Hz, eff_array, *p0): zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - full_spectrum = self.make_spectrum_composite_gaussian_lorentzian(survival_prob, sigma) + full_spectrum = self.make_spectrum_composite_gaussian_lorentzian(sigma) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f_intermediate = f_intermediate*eff_array f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) @@ -1300,8 +1301,8 @@ def fit_data_composite_gaussian_lorentzian(self, freq_bins, data_hist_freq, prin gaussian_portion_min = 1e-5 gaussian_portion_max = 1 N = len(self.gases) - p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess, sigma_guess] - p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (FWHM_eV_min, FWHM_eV_max)] + p0_guess = [B_field_guess, amplitude_guess, sigma_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (FWHM_eV_min, FWHM_eV_max)] # Actually do the fitting m_binned = Minuit.from_array_func(lambda p: self.chi_2_Poisson_composite_gaussian_lorentzian_reso(bins_Hz, data_hist_freq, eff_array, p), start = p0_guess, @@ -1313,15 +1314,13 @@ def fit_data_composite_gaussian_lorentzian(self, freq_bins, data_hist_freq, prin B_field_fit = params[0] #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) amplitude_fit = params[1] - survival_prob_fit = params[2] - sigma_fit = params[3] + sigma_fit = params[2] total_counts_fit = amplitude_fit perr = m_binned.np_errors() B_field_fit_err = perr[0] amplitude_fit_err = perr[1] - survival_prob_fit_err = perr[2] - sigma_fit_err = perr[3] + sigma_fit_err = perr[2] total_counts_fit_err = amplitude_fit_err fit_Hz = self.spectrum_func_composite_gaussian_lorentzian(bins_Hz, eff_array, *params) @@ -1338,9 +1337,6 @@ def fit_data_composite_gaussian_lorentzian(self, freq_bins, data_hist_freq, prin output_string += '-----------------\n' output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' output_string += '-----------------\n' - output_string += 'Survival probability \n= ' + "{:.2e}".format(survival_prob_fit)\ - +' +/- ' + "{:.2e}".format(survival_prob_fit_err)+'\n' - output_string += '-----------------\n' output_string += 'sigma = {:.2e}'.format(sigma_fit) + ' +/- {:.4e}\n'.format(sigma_fit_err) output_string += '-----------------\n' elapsed = time.time() - t @@ -1354,8 +1350,6 @@ def fit_data_composite_gaussian_lorentzian(self, freq_bins, data_hist_freq, prin 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, - 'survival_prob_fit': survival_prob_fit, - 'survival_prob_fit_err': survival_prob_fit_err, 'sigma_fit': sigma_fit, 'sigma_fit_err': sigma_fit_err, 'amplitude_fit': amplitude_fit, diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index e8437dc9..66c8a901 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -29,13 +29,14 @@ def test_complex_lineshape(self): } complexLineShape_config = { 'bins_choice': np.linspace(0e6, 100e6, 1000), - 'gases': ["H2", "Kr"], + 'gases': ["H2", "He"], 'max_scatters': 20, 'fixed_scatter_proportion': True, # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution 'resolution_function': 'gaussian_lorentzian_composite_resolution', # When fix_scatter_proportion is True, set the scatter proportion for gas1 below 'gas_scatter_proportion': [0.61, 0.39], + 'survival_prob': 0.8, # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, From 8227397d8b1edd7588416d64df199fab042e6479 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 4 Nov 2020 10:08:19 -0500 Subject: [PATCH 050/199] fixed survival probability free scatter proportion --- .../misc/MultiGasComplexLineShape.py | 523 +++++++++++++++++- test_analysis/Complex_line_shape_fitter.py | 12 +- 2 files changed, 516 insertions(+), 19 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 827554a7..75c83797 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -55,7 +55,15 @@ def InternalConfigure(self, params): self.fixed_scatter_proportion = reader.read_param(params, 'fixed_scatter_proportion', True) if self.fixed_scatter_proportion == True: self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) - self.survival_prob = reader.read_param(params, 'survival_prob', 1) + self.partially_fixed_scatter_proportion = reader.read_param(params, 'partially_fixed_scatter_proportion', True) + if self.partially_fixed_scatter_proportion == True: + self.free_gases = reader.read_param(params, 'free_gases', ["H2", "He"]) + self.fixed_gases = reader.read_param(params, 'fixed_gases', ["Ar", "Kr"]) + self.gases = self.free_gases + self.fixed_gases + self.scatter_proportion_for_fixed_gases = reader.read_param(params, 'scatter_proportion_for_fixed_gases', [0.018, 0.039]) + self.fixed_survival_probability = reader.read_param(params, 'fixed_survival_probability', True) + if self.fixed_survival_probability == True: + self.survival_prob = reader.read_param(params, 'survival_prob', 1) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution self.resolution_function = reader.read_param(params, 'resolution_function', '') @@ -94,18 +102,25 @@ def InternalRun(self): # guess = np.where(np.array(histogram) == np.max(histogram))[0][0] # kr17kev_in_hz = guess*(bins[1]-bins[0])+bins[0] #self.B_field = B(17.8, kr17kev_in_hz + 0) - if self.fixed_scatter_proportion == True: - if self.resolution_function == 'simulated_resolution': + if self.resolution_function == 'simulated_resolution': + if self.fixed_scatter_proportion == True: self.results = self.fit_data_ftc(freq_bins, data_hist_freq) - elif self.resolution_function == 'gaussian_resolution': - self.results = self.fit_data_1(freq_bins, data_hist_freq) - elif self.resolution_function == 'gaussian_lorentzian_composite_resolution': - self.results = self.fit_data_composite_gaussian_lorentzian(freq_bins, data_hist_freq) - else: - if self.resolution_function == 'simulated_resolution': + else: self.results = self.fit_data_ftc_2(freq_bins, data_hist_freq) - elif self.resolution_function == 'gaussian_resolution': + elif self.resolution_function == 'gaussian_resolution': + if self.fixed_scatter_proportion == True: + self.results = self.fit_data_1(freq_bins, data_hist_freq) + else: self.results = self.fit_data(freq_bins, data_hist_freq) + elif self.resolution_function == 'gaussian_lorentzian_composite_resolution': + if self.fixed_scatter_proportion == True and self.fixed_survival_probability == True: + self.results = self.fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(freq_bins, data_hist_freq) + elif self.fixed_scatter_proportion == True and self.fixed_survival_probability == False: + self.results = self.fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion(freq_bins, data_hist_freq) + elif self.fixed_scatter_proportion == False and self.fixed_survival_probability == True: + self.results = self.fit_data_composite_gaussian_lorentzian_fixed_survival_probability(freq_bins, data_hist_freq) + elif self.partially_fixed_scatter_proportion == True and self.fixed_survival_probability == True: + self.results = self.fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(freq_bins, data_hist_freq) return True @@ -391,7 +406,14 @@ def chi_2_Poisson_composite_gaussian_lorentzian_reso(self, bin_centers, data_his nonzero_bins_index = np.where(data_hist_freq != 0) zero_bins_index = np.where(data_hist_freq == 0) # expectation - fit_Hz = self.spectrum_func_composite_gaussian_lorentzian(bin_centers, eff_array, *params) + if self.fixed_scatter_proportion == True and self.fixed_survival_probability == True: + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_survival_probability(bin_centers, eff_array, *params) + elif self.fixed_scatter_proportion == True and self.fixed_survival_probability == False: + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion(bin_centers, eff_array, *params) + elif self.fixed_scatter_proportion == False and self.fixed_survival_probability == True: + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability(bin_centers, eff_array, *params) + elif self.partially_fixed_scatter_proportion == True and self.fixed_survival_probability == True: + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(bin_centers, eff_array, *params) chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() return chi2 @@ -1209,7 +1231,162 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print } return dictionary_of_fit_results - def make_spectrum_composite_gaussian_lorentzian(self, sigma, emitted_peak='shake'): + def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion(self, survival_prob, sigma, emitted_peak='shake'): + p = self.scatter_proportion + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_composite_gaussian_lorentzian(current_working_spectrum, sigma) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + relative_reconstruction_eff = np.exp(-0.643*M**0.74) + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + for i in range(0, M+1): + coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M + return current_full_spectrum + + def spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): + + B_field = p0[0] + amplitude = p0[1] + survival_prob = p0[2] + sigma = p0[3] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_composite_gaussian_lorentzian(survival_prob, sigma) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + FWHM_eV_guess = 5 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + sigma_guess = 5 + gamma_guess = 3 + gaussian_portion_guess = 0.5 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + FWHM_eV_min = 0 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + mu_min = -FWHM_eV_max + mu_max = FWHM_eV_max + gaussian_portion_min = 1e-5 + gaussian_portion_max = 1 + N = len(self.gases) + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess, sigma_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (FWHM_eV_min, FWHM_eV_max)] + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi_2_Poisson_composite_gaussian_lorentzian_reso(bins_Hz, data_hist_freq, eff_array, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + survival_prob_fit = params[2] + sigma_fit = params[3] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + survival_prob_fit_err = perr[2] + sigma_fit_err = perr[3] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian(bins_Hz, eff_array, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + + if print_params == True: + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Survival probability = {:.2e}'.format(survival_prob_fit) + ' +/- {:.4e}\n'.format(survival_prob_fit_err) + output_string += '-----------------\n' + output_string += 'sigma = {:.2e}'.format(sigma_fit) + ' +/- {:.4e}\n'.format(sigma_fit_err) + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'sigma_fit': sigma_fit, + 'sigma_fit_err': sigma_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results + + + def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(self, sigma, emitted_peak='shake'): p = self.scatter_proportion survival_prob = self.survival_prob scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') @@ -1243,7 +1420,7 @@ def make_spectrum_composite_gaussian_lorentzian(self, sigma, emitted_peak='shake current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum - def spectrum_func_composite_gaussian_lorentzian(self, bins_Hz, eff_array, *p0): + def spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(self, bins_Hz, eff_array, *p0): B_field = p0[0] amplitude = p0[1] @@ -1267,7 +1444,7 @@ def spectrum_func_composite_gaussian_lorentzian(self, bins_Hz, eff_array, *p0): return f - def fit_data_composite_gaussian_lorentzian(self, freq_bins, data_hist_freq, print_params=True): + def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(self, freq_bins, data_hist_freq, print_params=True): t = time.time() self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN @@ -1358,3 +1535,321 @@ def fit_data_composite_gaussian_lorentzian(self, freq_bins, data_hist_freq, prin 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results + + def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability(self, scatter_proportion, sigma, emitted_peak='shake'): + p = np.zeros(len(self.gases)) + p[0:-1] = scatter_proportion + p[-1] = 1 - sum(scatter_proportion) + survival_prob = self.survival_prob + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_composite_gaussian_lorentzian(current_working_spectrum, sigma) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + relative_reconstruction_eff = np.exp(-0.643*M**0.74) + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + for i in range(0, M+1): + coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M + return current_full_spectrum + + def spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability(self, bins_Hz, eff_array, *p0): + + B_field = p0[0] + amplitude = p0[1] + sigma = p0[2] + N = len(self.gases) + scatter_proportion = p0[3: 2+N] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability(scatter_proportion, sigma) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def fit_data_composite_gaussian_lorentzian_fixed_survival_probability(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + FWHM_eV_guess = 5 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + sigma_guess = 5 + gamma_guess = 3 + gaussian_portion_guess = 0.5 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + FWHM_eV_min = 0 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + mu_min = -FWHM_eV_max + mu_max = FWHM_eV_max + gaussian_portion_min = 1e-5 + gaussian_portion_max = 1 + N = len(self.gases) + p0_guess = [B_field_guess, amplitude_guess, sigma_guess] + (N-1)*[scatter_proportion_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (FWHM_eV_min, FWHM_eV_max)] + (N-1)*[(scatter_proportion_min, scatter_proportion_max)] + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi_2_Poisson_composite_gaussian_lorentzian_reso(bins_Hz, data_hist_freq, eff_array, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + sigma_fit = params[2] + scatter_proportion_fit = list(params[3:2+N]) + [1- sum(params[3:2+N])] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + sigma_fit_err = perr[2] + scatter_proportion_fit_err = list(perr[3:2+N]) + [np.sqrt(sum(perr[3:2+N]**2))] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability(bins_Hz, eff_array, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + + if print_params == True: + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'sigma = {:.2e}'.format(sigma_fit) + ' +/- {:.4e}\n'.format(sigma_fit_err) + output_string += '-----------------\n' + for i in range(len(self.gases)): + output_string += '{} Scatter proportion \n= '.format(self.gases[i]) + "{:.6e}".format(scatter_proportion_fit[i])\ + +' +/- ' + "{:.2e}".format(scatter_proportion_fit_err[i])+'\n' + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'sigma_fit': sigma_fit, + 'sigma_fit_err': sigma_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results + + def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(self, scatter_proportion, sigma, emitted_peak='shake'): + p = scatter_proportion + tuple([1-sum(scatter_proportion)]) + tuple(self.scatter_proportion_for_fixed_gases) + survival_prob = self.survival_prob + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_composite_gaussian_lorentzian(current_working_spectrum, sigma) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + relative_reconstruction_eff = np.exp(-0.643*M**0.74) + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + for i in range(0, M+1): + coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M + return current_full_spectrum + + def spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): + + B_field = p0[0] + amplitude = p0[1] + sigma = p0[2] + N = len(self.free_gases) + scatter_proportion = p0[3: 2+N] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability(scatter_proportion, sigma) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + FWHM_eV_guess = 5 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + sigma_guess = 5 + gamma_guess = 3 + gaussian_portion_guess = 0.5 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + FWHM_eV_min = 0 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + mu_min = -FWHM_eV_max + mu_max = FWHM_eV_max + gaussian_portion_min = 1e-5 + gaussian_portion_max = 1 + N = len(self.free_gases) + p0_guess = [B_field_guess, amplitude_guess, sigma_guess] + (N-1)*[scatter_proportion_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (FWHM_eV_min, FWHM_eV_max)] + (N-1)*[(scatter_proportion_min, scatter_proportion_max)] + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi_2_Poisson_composite_gaussian_lorentzian_reso(bins_Hz, data_hist_freq, eff_array, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + sigma_fit = params[2] + scatter_proportion_fit = list(params[3:2+N]) + [1- sum(params[3:2+N])] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + sigma_fit_err = perr[2] + scatter_proportion_fit_err = list(perr[3:2+N]) + [np.sqrt(sum(perr[3:2+N]**2))] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability(bins_Hz, eff_array, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + + if print_params == True: + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'sigma = {:.2e}'.format(sigma_fit) + ' +/- {:.4e}\n'.format(sigma_fit_err) + output_string += '-----------------\n' + for i in range(len(self.free_gases)): + output_string += '{} Scatter proportion \n= '.format(self.free_gases[i]) + "{:.6e}".format(scatter_proportion_fit[i])\ + +' +/- ' + "{:.2e}".format(scatter_proportion_fit_err[i])+'\n' + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'sigma_fit': sigma_fit, + 'sigma_fit_err': sigma_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 66c8a901..88c86583 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -31,12 +31,14 @@ def test_complex_lineshape(self): 'bins_choice': np.linspace(0e6, 100e6, 1000), 'gases': ["H2", "He"], 'max_scatters': 20, - 'fixed_scatter_proportion': True, + 'fixed_scatter_proportion': False, + 'fixed_survival_probability': True, # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution 'resolution_function': 'gaussian_lorentzian_composite_resolution', - # When fix_scatter_proportion is True, set the scatter proportion for gas1 below - 'gas_scatter_proportion': [0.61, 0.39], - 'survival_prob': 0.8, + # When fixed_scatter_proportion is True, set the scatter proportion for the gases below + 'gas_scatter_proportion': [0.6, 0.4], + # When option fixed_survival_probability is True, assign the survival probability below + 'survival_prob': 10/11., # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -85,7 +87,7 @@ def test_complex_lineshape(self): str_gas_scatter_proportion += ': ' str_gas_scatter_proportion += str(complexLineShape_config['gas_scatter_proportion'][i]) str_gas_scatter_proportion += ' ' - plot_title += '\n with fixed scatter proportion \n {}'.format(str_gas_scatter_proportion) + #plot_title += '\n with fixed scatter proportion \n {}'.format(str_gas_scatter_proportion) plt.title(plot_title) plt.tight_layout() plt.savefig('/host/plots/fit_FTC_march_with_composite_gaussian_lorentzian_resolution.png'.format(len(complexLineShape_config['gases']))) From d70e4c012581281d7d01f037cc3a87373f3022d8 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 4 Nov 2020 22:46:58 -0500 Subject: [PATCH 051/199] add fitting with partially fixed scatter proportion --- .../processors/misc/MultiGasComplexLineShape.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 75c83797..9e6c8770 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -117,7 +117,7 @@ def InternalRun(self): self.results = self.fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(freq_bins, data_hist_freq) elif self.fixed_scatter_proportion == True and self.fixed_survival_probability == False: self.results = self.fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion(freq_bins, data_hist_freq) - elif self.fixed_scatter_proportion == False and self.fixed_survival_probability == True: + elif self.fixed_scatter_proportion == False and self.fixed_survival_probability == True and self.partially_fixed_scatter_proportion == False: self.results = self.fit_data_composite_gaussian_lorentzian_fixed_survival_probability(freq_bins, data_hist_freq) elif self.partially_fixed_scatter_proportion == True and self.fixed_survival_probability == True: self.results = self.fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(freq_bins, data_hist_freq) @@ -410,7 +410,7 @@ def chi_2_Poisson_composite_gaussian_lorentzian_reso(self, bin_centers, data_his fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_survival_probability(bin_centers, eff_array, *params) elif self.fixed_scatter_proportion == True and self.fixed_survival_probability == False: fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion(bin_centers, eff_array, *params) - elif self.fixed_scatter_proportion == False and self.fixed_survival_probability == True: + elif self.fixed_scatter_proportion == False and self.fixed_survival_probability == True and self.partially_fixed_scatter_proportion == False: fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability(bin_centers, eff_array, *params) elif self.partially_fixed_scatter_proportion == True and self.fixed_survival_probability == True: fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(bin_centers, eff_array, *params) @@ -1697,7 +1697,8 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability(self, freq return dictionary_of_fit_results def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(self, scatter_proportion, sigma, emitted_peak='shake'): - p = scatter_proportion + tuple([1-sum(scatter_proportion)]) + tuple(self.scatter_proportion_for_fixed_gases) + p = scatter_proportion + tuple([1-sum(scatter_proportion)-sum(self.scatter_proportion_for_fixed_gases)]) + tuple(self.scatter_proportion_for_fixed_gases) + logger.info(p) survival_prob = self.survival_prob scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) @@ -1749,7 +1750,7 @@ def spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability_parti zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - full_spectrum = self.make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability(scatter_proportion, sigma) + full_spectrum = self.make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(scatter_proportion, sigma) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f_intermediate = f_intermediate*eff_array f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) @@ -1804,7 +1805,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_ #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) amplitude_fit = params[1] sigma_fit = params[2] - scatter_proportion_fit = list(params[3:2+N]) + [1- sum(params[3:2+N])] + scatter_proportion_fit = list(params[3:2+N]) + [1- sum(params[3:2+N]) - sum(self.scatter_proportion_for_fixed_gases)] total_counts_fit = amplitude_fit perr = m_binned.np_errors() @@ -1834,6 +1835,9 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_ output_string += '{} Scatter proportion \n= '.format(self.free_gases[i]) + "{:.6e}".format(scatter_proportion_fit[i])\ +' +/- ' + "{:.2e}".format(scatter_proportion_fit_err[i])+'\n' output_string += '-----------------\n' + for i in range(len(self.fixed_gases)): + output_string += '{} Scatter proportion (fixed) \n= '.format(self.fixed_gases[i]) + "{:.6e}".format(self.scatter_proportion_for_fixed_gases[i]) + output_string += '-----------------\n' elapsed = time.time() - t output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' dictionary_of_fit_results = { From a94dfc512219e80e3863f8f8e7d8fa89fad23077 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Fri, 6 Nov 2020 10:32:50 -0500 Subject: [PATCH 052/199] update fitting with fixed survival probability partially fixed scatter proportion --- mermithid/processors/misc/MultiGasComplexLineShape.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 9e6c8770..5e3416fd 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -1815,7 +1815,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_ scatter_proportion_fit_err = list(perr[3:2+N]) + [np.sqrt(sum(perr[3:2+N]**2))] total_counts_fit_err = amplitude_fit_err - fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability(bins_Hz, eff_array, *params) + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) @@ -1836,7 +1836,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_ +' +/- ' + "{:.2e}".format(scatter_proportion_fit_err[i])+'\n' output_string += '-----------------\n' for i in range(len(self.fixed_gases)): - output_string += '{} Scatter proportion (fixed) \n= '.format(self.fixed_gases[i]) + "{:.6e}".format(self.scatter_proportion_for_fixed_gases[i]) + output_string += '{} Scatter proportion (fixed) \n= '.format(self.fixed_gases[i]) + "{:.6e}\n".format(self.scatter_proportion_for_fixed_gases[i]) output_string += '-----------------\n' elapsed = time.time() - t output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' From 222933683b6e0161ee60d9e52bff2ec0e3c81622 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 16 Nov 2020 16:47:20 -0500 Subject: [PATCH 053/199] add mass 28 gases --- mermithid/misc/ComplexLineShapeUtilities.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/mermithid/misc/ComplexLineShapeUtilities.py b/mermithid/misc/ComplexLineShapeUtilities.py index 4fe05dc7..ad2502f1 100644 --- a/mermithid/misc/ComplexLineShapeUtilities.py +++ b/mermithid/misc/ComplexLineShapeUtilities.py @@ -39,9 +39,12 @@ def read_oscillator_str_file(filename): for line in lines: if line != "" and line[0]!="#": - raw_data = [float(i) for i in line.split("\t")] - energyOsc[0].append(raw_data[0]) - energyOsc[1].append(raw_data[1]) + try: + raw_data = [float(i) for i in line.split("\t")] + energyOsc[0].append(raw_data[0]) + energyOsc[1].append(raw_data[1]) + except: + continue energyOsc = np.array(energyOsc) ### take data and sort by energy @@ -62,6 +65,12 @@ def aseev_func_tail(energy_loss_array, gas_type): A2, omeg2, eps2 = 0.1187, 33.40, 10.43 elif gas_type=="Ar": A2, omeg2, eps2 = 0.3344, 21.91, 21.14 + elif gas_type=="N2": + A2, omeg2, eps2 = 0.21754816, 44.99897054, 20.43916114 + elif gas_type=="CO": + A2, omeg2, eps2 = 0.19583454, 55.21888452, 16.44972596 + elif gas_type=="C2H4": + A2, omeg2, eps2 = 0.57492182, 23.77501391, 14.33107345 return A2*omeg2**2./(omeg2**2.+4*(energy_loss_array-eps2)**2.) #convert oscillator strength into energy loss spectrum From 5fbb06f0ab353bcd0494ac83012ffa4ca10cc01f Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 16 Nov 2020 20:36:35 -0500 Subject: [PATCH 054/199] make gases and scatter proportion configurable --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 47aea896..8e99c661 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -100,7 +100,8 @@ def InternalConfigure(self, params): self.survival_prob = reader.read_param(params, 'survival_prob', 0.77) self.scattering_sigma = reader.read_param(params, 'scattering_sigma', 18.6) self.NScatters = reader.read_param(params, 'NScatters', 20) - self.scatter_proportion = reader.read_param(params, 'scatter_proportion', 1.0) + self.gases = reader.read_param(params, 'gases', ['H2', 'He']) + self.scatter_proportion = reader.read_param(params, 'scatter_proportion', []) #paths self.simplified_scattering_path = reader.read_param(params, 'simplified_scattering_path', '/host/input_data/simplified_scattering_params.txt') @@ -152,7 +153,7 @@ def InternalConfigure(self, params): 'max_scatters': self.NScatters, 'fixed_scatter_proportion': True, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below - 'gas_scatter_proportion': [self.scatter_proportion, 1.-self.scatter_proportion], + 'gas_scatter_proportion': self.scatter_proportion, #'gas1_scatter_proportion': self.scatter_proportion, #, 1.-self.scatter_proportion], 'use_simulated_inst_reso': True, 'use_combined_four_trap_inst_reso': False, From 2323155ece606b6eedc9a91d9e3a288540ab848a Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 16 Nov 2020 20:44:15 -0500 Subject: [PATCH 055/199] make gases and scatter_proportion configurable --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 8e99c661..9cc65b54 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -149,7 +149,7 @@ def InternalConfigure(self, params): # Setup and configure lineshape processor complexLineShape_config = { - 'gases': ["H2", "He"], + 'gases': self.gases, 'max_scatters': self.NScatters, 'fixed_scatter_proportion': True, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below From 6325498a44e4be3c02070be836013fd4d6c7841b Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Tue, 17 Nov 2020 10:54:39 -0500 Subject: [PATCH 056/199] add elevated, composite, composite with pedestal factor gaussian resolutions --- .../misc/MultiGasComplexLineShape.py | 618 +++++++++++++++++- test_analysis/Complex_line_shape_fitter.py | 24 +- 2 files changed, 604 insertions(+), 38 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 5e3416fd..62f9c71f 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -67,6 +67,15 @@ def InternalConfigure(self, params): self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution self.resolution_function = reader.read_param(params, 'resolution_function', '') + if self.resolution_function == 'gaussian_lorentzian_composite_resolution': + self.ratio_gamma_to_sigma = reader.read_param(params, 'ratio_gamma_to_sigma', 0.8) + self.gaussian_proportion = reader.read_param(params, 'gaussian_proportion', 0.8) + if self.resolution_function == 'elevated_gaussian': + self.ratio_gamma_to_sigma = reader.read_param(params, 'ratio_gamma_to_sigma', 0.8) + if self.resolution_function == 'composite_gaussian' or 'composite_gaussian_pedestal_factor': + self.A_array = reader.read_param(params, 'A_array', [0.076, 0.341, 0.381, 0.203]) + self.sigma_array = reader.read_param(params, 'sigma_array', [5.01, 13.33, 15.40, 11.85]) + #self.elevation_factor = reader.read_param(params, 'elevation_factor', 20) # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown self.num_points_in_std_array = reader.read_param(params, 'num_points_in_std_array', 10000) @@ -121,6 +130,12 @@ def InternalRun(self): self.results = self.fit_data_composite_gaussian_lorentzian_fixed_survival_probability(freq_bins, data_hist_freq) elif self.partially_fixed_scatter_proportion == True and self.fixed_survival_probability == True: self.results = self.fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(freq_bins, data_hist_freq) + elif self.resolution_function == 'elevated_gaussian': + self.results = self.fit_data_elevated_gaussian_fixed_scatter_proportion(freq_bins, data_hist_freq) + elif self.resolution_function == 'composite_gaussian': + self.results = self.fit_data_composite_gaussian_fixed_scatter_proportion(freq_bins, data_hist_freq) + elif self.resolution_function == 'composite_gaussian_pedestal_factor': + self.results = self.fit_data_composite_gaussian_pedestal_factor_fixed_scatter_proportion(freq_bins, data_hist_freq) return True @@ -150,10 +165,22 @@ def std_gaussian(self, sigma): x_array = self.std_eV_array() ans = ComplexLineShapeUtilities.gaussian(x_array,1,sigma,0) return ans - - def composite_gaussian(self, A_array, sigma_array): + + def composite_gaussian(self): + x_array = self.std_eV_array() + ans = 0 + A_array = self.A_array + sigma_array = self.sigma_array + for A, sigma in zip(A_array, sigma_array): + ans += self.gaussian(x_array, A, sigma, 0) + return ans + + def composite_gaussian_pedestal_factor(self, pedestal_factor): x_array = self.std_eV_array() ans = 0 + A_array = self.A_array + sigma_array = np.array(self.sigma_array) + sigma_array = sigma_array*[1, 1, pedestal_factor, 1] for A, sigma in zip(A_array, sigma_array): ans += self.gaussian(x_array, A, sigma, 0) return ans @@ -190,14 +217,24 @@ def std_smeared_triangle(self, center, scale1, scale2, exponent, sigma): def composite_gaussian_lorentzian(self, sigma): x_array = self.std_eV_array() w_g = x_array/sigma - gamma = 0.8*sigma + gamma = self.ratio_gamma_to_sigma*sigma w_l = x_array/gamma lorentzian = 1./(gamma*np.pi)*1./(1+(w_l**2)) gaussian = 1./(np.sqrt(2.*np.pi)*sigma)*np.exp(-0.5*w_g**2) - p = 0.8 + p = self.gaussian_proportion composite_function = p*gaussian+(1-p)*lorentzian return composite_function + def elevated_gaussian(self, elevation_factor, sigma): + x_array = self.std_eV_array() + w_g = x_array/sigma + gamma = self.ratio_gamma_to_sigma*sigma + w_l = x_array/gamma + lorentzian = 1./(gamma*np.pi)*1./(1+(w_l**2)) + gaussian = 1./(np.sqrt(2.*np.pi)*sigma)*np.exp(-0.5*w_g**2) + modified_guassian_function = gaussian*(1 + elevation_factor*lorentzian) + return modified_guassian_function + # normalizes a function, but depends on binning. # Only to be used for functions evaluated on the SELA def normalize(self, f): @@ -347,6 +384,24 @@ def convolve_composite_gaussian_lorentzian(self, func_to_convolve, sigma): ans_normed = self.normalize(ans) return ans_normed + def convolve_elevated_gaussian(self, func_to_convolve, elevation_factor, sigma): + resolution_f = self.elevated_gaussian(elevation_factor, sigma) + ans = signal.convolve(resolution_f, func_to_convolve, mode = 'same') + ans_normed = self.normalize(ans) + return ans_normed + + def convolve_composite_gaussian(self, func_to_convolve): + resolution_f = self.composite_gaussian() + ans = signal.convolve(resolution_f, func_to_convolve, mode = 'same') + ans_normed = self.normalize(ans) + return ans_normed + + def convolve_composite_gaussian_pedestal_factor(self, func_to_convolve, pedestal_factor): + resolution_f = self.composite_gaussian_pedestal_factor(pedestal_factor) + ans = signal.convolve(resolution_f, func_to_convolve, mode = 'same') + ans_normed = self.normalize(ans) + return ans_normed + def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt) x_data = ins_resolution_data.T[0] @@ -388,14 +443,14 @@ def chi2_Poisson(self, bin_centers, data_hist_freq, params): nonzero_bins_index = np.where(data_hist_freq != 0) zero_bins_index = np.where(data_hist_freq == 0) # expectation - if self.fixed_scatter_proportion: - if self.fit_ftc: + if self.resolution_function == 'simulated_resolution': + if self.fixed_scatter_proportion: fit_Hz = self.spectrum_func_ftc(bin_centers, *params) else: - fit_Hz = self.spectrum_func_1(bin_centers, *params) - else: - if self.fit_ftc: fit_Hz = self.spectrum_func_ftc_2(bin_centers, *params) + if self.resolution_function == 'gaussian_resolution': + if self.fixed_scatter_proportion: + fit_Hz = self.spectrum_func_1(bin_centers, *params) else: fit_Hz = self.spectrum_func(bin_centers, *params) chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() @@ -418,6 +473,33 @@ def chi_2_Poisson_composite_gaussian_lorentzian_reso(self, bin_centers, data_his chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() return chi2 + def chi_2_Poisson_elevated_gaussian_reso(self, bin_centers, data_hist_freq, eff_array, params): + nonzero_bins_index = np.where(data_hist_freq != 0) + zero_bins_index = np.where(data_hist_freq == 0) + # expectation + fit_Hz = self.spectrum_func_elevated_gaussian_fixed_scatter_proportion(bin_centers, eff_array, *params) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + return chi2 + + def chi_2_Poisson_composite_gaussian_reso(self, bin_centers, data_hist_freq, eff_array, params): + nonzero_bins_index = np.where(data_hist_freq != 0) + zero_bins_index = np.where(data_hist_freq == 0) + # expectation + fit_Hz = self.spectrum_func_composite_gaussian_fixed_scatter_proportion(bin_centers, eff_array, *params) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + return chi2 + + def chi_2_Poisson_composite_gaussian_pedestal_factor_reso(self, bin_centers, data_hist_freq, eff_array, params): + nonzero_bins_index = np.where(data_hist_freq != 0) + zero_bins_index = np.where(data_hist_freq == 0) + # expectation + fit_Hz = self.spectrum_func_composite_gaussian_pedestal_factor_fixed_scatter_proportion(bin_centers, eff_array, *params) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + return chi2 + def reduced_chi2_Pearson_Neyman_composite(self, data_hist_freq, fit_Hz, number_of_parameters): nonzero_bins_index = np.where(data_hist_freq != 0)[0] zero_bins_index = np.where(data_hist_freq == 0)[0] @@ -765,7 +847,8 @@ def fit_data_1(self, freq_bins, data_hist_freq): } return dictionary_of_fit_results - def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): + def make_spectrum_ftc(self, survival_prob, emitted_peak='shake'): + logger.info(survival_prob) gases = self.gases current_path = self.path_to_scatter_spectra_file # check_existence_of_scatter_files() @@ -773,9 +856,7 @@ def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): p = np.zeros(len(gases)) p = self.scatter_proportion scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') - scatter_spectra = np.load( - scatter_spectra_file_path, allow_pickle = True - ) + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) en_array = self.std_eV_array() current_full_spectrum = np.zeros(len(en_array)) if emitted_peak == 'lorentzian': @@ -787,6 +868,7 @@ def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): current_full_spectrum += current_working_spectrum N = len(self.gases) for M in range(1, self.max_scatters + 1): + relative_reconstruction_eff = np.exp(-0.643*M**0.74) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: entry_str = '' @@ -797,29 +879,30 @@ def make_spectrum_ftc(self, prob_parameter, emitted_peak='shake'): current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) coefficient = factorial(sum(combination)) for component, i in zip(combination, range(len(self.gases))): - coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M - current_full_spectrum += coefficient*current_working_spectrum + coefficient = coefficient/factorial(component)*p[i]**component + for i in range(0, M+1): + coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum def spectrum_func_ftc(self, bins_Hz, *p0): B_field = p0[0] amplitude = p0[1] - prob_parameter = p0[2] - + survival_prob = p0[2] x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + logger.info(B_field) en_loss_array = self.std_eV_array() en_loss_array_min = en_loss_array[0] en_loss_array_max = en_loss_array[len(en_loss_array)-1] - en_array_rev = ComplexLineShapeUtilities.flip_array(-1*en_loss_array) f = np.zeros(len(x_eV)) f_intermediate = np.zeros(len(x_eV)) x_eV_minus_line = Constants.kr_k_line_e() - x_eV - zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0], np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - full_spectrum = self.make_spectrum_ftc(prob_parameter) - f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx],en_array_rev,full_spectrum) + full_spectrum = self.make_spectrum_ftc(survival_prob) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) return f @@ -832,7 +915,7 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): # Initial guesses for curve_fit B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) amplitude_guess = np.sum(data_hist_freq)/2 - prob_parameter_guess = 0.5 + prob_parameter_guess = 0.99 # Bounds for curve_fit B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) @@ -855,13 +938,13 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): ################### Generalize to N Gases ########################### B_field_fit = params[0] amplitude_fit = params[1] - prob_parameter_fit = params[2] + survival_prob_fit = params[2] total_counts_fit = amplitude_fit perr = m_binned.np_errors() B_field_fit_err = perr[0] amplitude_fit_err = perr[1] - prob_parameter_fit_err = perr[2] + survival_prob_fit_err = perr[2] total_counts_fit_err = amplitude_fit_err fit_Hz = self.spectrum_func_ftc(bins_Hz, *params) @@ -878,8 +961,8 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): output_string += '-----------------\n' output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' output_string += '-----------------\n' - output_string += 'Probability parameter \n= ' + "{:.2e}".format(prob_parameter_fit)\ - +' +/- ' + "{:.2e}".format(prob_parameter_fit_err)+'\n' + output_string += 'Survival probability \n= ' + "{:.8e}".format(survival_prob_fit)\ + +' +/- ' + "{:.6e}".format(survival_prob_fit_err)+'\n' output_string += '-----------------\n' output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' dictionary_of_fit_results = { @@ -891,8 +974,8 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, - 'prob_parameter_fit': prob_parameter_fit, - 'prob_parameter_fit_err': prob_parameter_fit_err, + 'survival_prob_fit': survival_prob_fit, + 'survival_prob_fit_err': survival_prob_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, 'data_hist_freq': data_hist_freq, @@ -1282,7 +1365,7 @@ def spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion(self, b zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - full_spectrum = self.make_spectrum_composite_gaussian_lorentzian(survival_prob, sigma) + full_spectrum = self.make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion(survival_prob, sigma) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f_intermediate = f_intermediate*eff_array f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) @@ -1347,7 +1430,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion(self, freq_b sigma_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - fit_Hz = self.spectrum_func_composite_gaussian_lorentzian(bins_Hz, eff_array, *params) + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) @@ -1361,7 +1444,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion(self, freq_b output_string += '-----------------\n' output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' output_string += '-----------------\n' - output_string += 'Survival probability = {:.2e}'.format(survival_prob_fit) + ' +/- {:.4e}\n'.format(survival_prob_fit_err) + output_string += 'Survival probability = {:.8e}'.format(survival_prob_fit) + ' +/- {:.8e}\n'.format(survival_prob_fit_err) output_string += '-----------------\n' output_string += 'sigma = {:.2e}'.format(sigma_fit) + ' +/- {:.4e}\n'.format(sigma_fit_err) output_string += '-----------------\n' @@ -1838,6 +1921,12 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_ for i in range(len(self.fixed_gases)): output_string += '{} Scatter proportion (fixed) \n= '.format(self.fixed_gases[i]) + "{:.6e}\n".format(self.scatter_proportion_for_fixed_gases[i]) output_string += '-----------------\n' + output_string += 'Survival probability (fixed) = {:2e}\n'.format(self.survival_prob) + output_string += '-----------------\n' + output_string += 'Gaussian + Lorentzian resolution:\n' + output_string += ' ratio of gamma to sigma (fixed) = {:2e}\n'.format(self.ratio_gamma_to_sigma) + output_string += ' gaussian proportion (fixed) = {:2e}\n'.format(self.gaussian_proportion) + output_string += '-----------------\n' elapsed = time.time() - t output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' dictionary_of_fit_results = { @@ -1857,3 +1946,470 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_ 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results + + def make_spectrum_elevated_gaussian_fixed_scatter_proportion(self, survival_prob, sigma, elevation_factor, emitted_peak='shake'): + p = self.scatter_proportion + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_elevated_gaussian(current_working_spectrum, elevation_factor, sigma) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + relative_reconstruction_eff = np.exp(-0.643*M**0.74) + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + for i in range(0, M+1): + coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M + return current_full_spectrum + + def spectrum_func_elevated_gaussian_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): + + B_field = p0[0] + amplitude = p0[1] + survival_prob = p0[2] + sigma = p0[3] + elevation_factor = p0[4] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_elevated_gaussian_fixed_scatter_proportion(survival_prob, sigma, elevation_factor) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def fit_data_elevated_gaussian_fixed_scatter_proportion(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + FWHM_eV_guess = 5 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + sigma_guess = 5 + gamma_guess = 3 + gaussian_portion_guess = 0.5 + elevation_factor_guess = 20 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + FWHM_eV_min = 0 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + mu_min = -FWHM_eV_max + mu_max = FWHM_eV_max + gaussian_portion_min = 1e-5 + gaussian_portion_max = 1 + elevation_factor_min = 0 + elevation_factor_max = 500 + N = len(self.gases) + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess, sigma_guess, elevation_factor_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (FWHM_eV_min, FWHM_eV_max), (elevation_factor_min, elevation_factor_max)] + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi_2_Poisson_elevated_gaussian_reso(bins_Hz, data_hist_freq, eff_array, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + survival_prob_fit = params[2] + sigma_fit = params[3] + elevation_factor_fit = params[4] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + survival_prob_fit_err = perr[2] + sigma_fit_err = perr[3] + elevation_factor_fit_err = perr[4] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_elevated_gaussian_fixed_scatter_proportion(bins_Hz, eff_array, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + + if print_params == True: + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Survival probability = {:.8e}'.format(survival_prob_fit) + ' +/- {:.8e}\n'.format(survival_prob_fit_err) + output_string += '-----------------\n' + output_string += 'sigma = {:.2e}'.format(sigma_fit) + ' +/- {:.4e}\n'.format(sigma_fit_err) + output_string += '-----------------\n' + output_string += 'elevation factor = {:.2e}'.format(elevation_factor_fit) + ' +/- {:.4e}\n'.format(elevation_factor_fit_err) + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'sigma_fit': sigma_fit, + 'sigma_fit_err': sigma_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results + + def make_spectrum_composite_gaussian_fixed_scatter_proportion(self, survival_prob, emitted_peak='shake'): + p = self.scatter_proportion + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_composite_gaussian(current_working_spectrum) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + relative_reconstruction_eff = np.exp(-0.643*M**0.74) + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + for i in range(0, M+1): + coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M + return current_full_spectrum + + def spectrum_func_composite_gaussian_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): + + B_field = p0[0] + amplitude = p0[1] + survival_prob = p0[2] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_composite_gaussian_fixed_scatter_proportion(survival_prob) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def fit_data_composite_gaussian_fixed_scatter_proportion(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + FWHM_eV_guess = 5 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + sigma_guess = 5 + gamma_guess = 3 + gaussian_portion_guess = 0.5 + elevation_factor_guess = 20 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + FWHM_eV_min = 0 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + mu_min = -FWHM_eV_max + mu_max = FWHM_eV_max + gaussian_portion_min = 1e-5 + gaussian_portion_max = 1 + elevation_factor_min = 0 + elevation_factor_max = 500 + N = len(self.gases) + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max)] + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi_2_Poisson_composite_gaussian_reso(bins_Hz, data_hist_freq, eff_array, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + survival_prob_fit = params[2] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + survival_prob_fit_err = perr[2] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_composite_gaussian_fixed_scatter_proportion(bins_Hz, eff_array, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + + if print_params == True: + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Survival probability = {:.8e}'.format(survival_prob_fit) + ' +/- {:.8e}\n'.format(survival_prob_fit_err) + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results + + def make_spectrum_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, survival_prob, pedestal_factor, emitted_peak='shake'): + p = self.scatter_proportion + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_composite_gaussian_pedestal_factor(current_working_spectrum, pedestal_factor) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + relative_reconstruction_eff = np.exp(-0.643*M**0.74) + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + for i in range(0, M+1): + coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M + return current_full_spectrum + + def spectrum_func_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): + + B_field = p0[0] + amplitude = p0[1] + survival_prob = p0[2] + pedestal_factor = p0[3] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_composite_gaussian_pedestal_factor_fixed_scatter_proportion(survival_prob, pedestal_factor) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def fit_data_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + FWHM_eV_guess = 5 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + sigma_guess = 5 + gamma_guess = 3 + gaussian_portion_guess = 0.5 + pedestal_factor_guess = 1. + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + FWHM_eV_min = 0 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + mu_min = -FWHM_eV_max + mu_max = FWHM_eV_max + gaussian_portion_min = 1e-5 + gaussian_portion_max = 1 + pedestal_factor_min = 1e-5 + pedestal_factor_max = 500 + N = len(self.gases) + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess, pedestal_factor_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (pedestal_factor_min, pedestal_factor_max)] + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi_2_Poisson_composite_gaussian_pedestal_factor_reso(bins_Hz, data_hist_freq, eff_array, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + survival_prob_fit = params[2] + pedestal_factor_fit = params[3] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + survival_prob_fit_err = perr[2] + pedestal_factor_fit_err = perr[3] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_composite_gaussian_pedestal_factor_fixed_scatter_proportion(bins_Hz, eff_array, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + + if print_params == True: + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Survival probability = {:.8e}'.format(survival_prob_fit) + ' +/- {:.8e}\n'.format(survival_prob_fit_err) + output_string += '-----------------\n' + output_string += 'pedestal factor = {:.8e}'.format(pedestal_factor_fit) + ' +/- {:.8e}\n'.format(pedestal_factor_fit_err) + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results \ No newline at end of file diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 88c86583..116d2ea9 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -31,14 +31,24 @@ def test_complex_lineshape(self): 'bins_choice': np.linspace(0e6, 100e6, 1000), 'gases': ["H2", "He"], 'max_scatters': 20, - 'fixed_scatter_proportion': False, - 'fixed_survival_probability': True, - # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution - 'resolution_function': 'gaussian_lorentzian_composite_resolution', + 'fixed_scatter_proportion': True, # When fixed_scatter_proportion is True, set the scatter proportion for the gases below - 'gas_scatter_proportion': [0.6, 0.4], + 'gas_scatter_proportion': [0.75, 0.25],#0.753, 0.190, 0.018, 0.039 + 'partially_fixed_scatter_proportion': False, + 'free_gases': ["H2", "He"], + 'fixed_gases': ["Ar", "Kr"], + 'scatter_proportion_for_fixed_gases': [0.018, 0.039], + 'fixed_survival_probability': False, # When option fixed_survival_probability is True, assign the survival probability below - 'survival_prob': 10/11., + 'survival_prob': 15/16., # assuming total cross section for elastic scattering is 1/10 of inelastic scattering + # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution + 'resolution_function': 'composite_gaussian_pedestal_factor', + # specific choice of parameters in the gaussian lorentzian composite resolution function + 'ratio_gamma_to_sigma': 0.8, + 'gaussian_proportion': 1., + # if the resolution function is composite gaussian + 'sigma_array': [5.01, 13.33, 25, 11.85], + 'A_array': [0.076, 0.341, 0.381, 0.203], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -90,7 +100,7 @@ def test_complex_lineshape(self): #plot_title += '\n with fixed scatter proportion \n {}'.format(str_gas_scatter_proportion) plt.title(plot_title) plt.tight_layout() - plt.savefig('/host/plots/fit_FTC_march_with_composite_gaussian_lorentzian_resolution.png'.format(len(complexLineShape_config['gases']))) + plt.savefig('/host/plots/fit_FTC_march_with_composite_gaussian_pedestal_factor_resolution.png'.format(len(complexLineShape_config['gases']))) if __name__ == '__main__': From bb46e17843d6c8951bcff15f75dd446552c3a1eb Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Tue, 24 Nov 2020 13:23:26 -0500 Subject: [PATCH 057/199] composite gaussian resolution scaled and simulated resolution scaled --- .../misc/MultiGasComplexLineShape.py | 363 ++++++++++++++++++ test_analysis/Complex_line_shape_fitter.py | 26 +- 2 files changed, 374 insertions(+), 15 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 62f9c71f..376eed76 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -136,6 +136,10 @@ def InternalRun(self): self.results = self.fit_data_composite_gaussian_fixed_scatter_proportion(freq_bins, data_hist_freq) elif self.resolution_function == 'composite_gaussian_pedestal_factor': self.results = self.fit_data_composite_gaussian_pedestal_factor_fixed_scatter_proportion(freq_bins, data_hist_freq) + elif self.resolution_function == 'composite_gaussian_scaled': + self.results = self.fit_data_composite_gaussian_scaled_fixed_scatter_proportion(freq_bins, data_hist_freq) + elif self.resolution_function == 'simulated_resolution_scaled': + self.results = self.fit_data_simulated_resolution_scaled_fixed_scatter_proportion(freq_bins, data_hist_freq) return True @@ -185,6 +189,16 @@ def composite_gaussian_pedestal_factor(self, pedestal_factor): ans += self.gaussian(x_array, A, sigma, 0) return ans + def composite_gaussian_scaled(self, scale_factor): + x_array = self.std_eV_array() + ans = 0 + A_array = self.A_array + sigma_array = np.array(self.sigma_array) + sigma_array = sigma_array*scale_factor + for A, sigma in zip(A_array, sigma_array): + ans += self.gaussian(x_array, A, sigma, 0) + return ans + def asym_triangle(self, x, scale1, scale2, center, exponent=1): index_below = np.where(x=center) @@ -402,11 +416,20 @@ def convolve_composite_gaussian_pedestal_factor(self, func_to_convolve, pedestal ans_normed = self.normalize(ans) return ans_normed + def convolve_composite_gaussian_scaled(self, func_to_convolve, scale_factor): + resolution_f = self.composite_gaussian_scaled(scale_factor) + ans = signal.convolve(resolution_f, func_to_convolve, mode = 'same') + ans_normed = self.normalize(ans) + return ans_normed + def read_ins_resolution_data(self, path_to_ins_resolution_data_txt): ins_resolution_data = np.loadtxt(path_to_ins_resolution_data_txt) x_data = ins_resolution_data.T[0] y_data = ins_resolution_data.T[1] y_err_data = ins_resolution_data.T[2] + x_data = ComplexLineShapeUtilities.flip_array(-1*x_data) + y_data = ComplexLineShapeUtilities.flip_array(y_data) + y_err_data = ComplexLineShapeUtilities.flip_array(y_err_data) return x_data, y_data, y_err_data def convolve_ins_resolution(self, working_spectrum): @@ -420,6 +443,18 @@ def convolve_ins_resolution(self, working_spectrum): normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum + def convolve_simulated_resolution_scaled(self, working_spectrum, scale_factor): + x_data, y_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + scaled_xdata = x_data*scale_factor + f = interpolate.interp1d(x_data*scale_factor, y_data) + x_array = self.std_eV_array() + y_array = np.zeros(len(x_array)) + index_within_range_of_xdata = np.where((x_array >= scaled_xdata[0]) & (x_array <= scaled_xdata[-1])) + y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]) + convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') + normalized_convolved_spectrum = self.normalize(convolved_spectrum) + return normalized_convolved_spectrum + def convolve_smeared_triangle(self, func_to_convolve, center, scale1, scale2, exponent, sigma): resolution_f = self.std_smeared_triangle(center, scale1, scale2, exponent, sigma) ans = signal.convolve(resolution_f, func_to_convolve, mode = 'same') @@ -500,6 +535,24 @@ def chi_2_Poisson_composite_gaussian_pedestal_factor_reso(self, bin_centers, dat chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() return chi2 + def chi_2_Poisson_composite_gaussian_scaled_reso(self, bin_centers, data_hist_freq, eff_array, params): + nonzero_bins_index = np.where(data_hist_freq != 0) + zero_bins_index = np.where(data_hist_freq == 0) + # expectation + fit_Hz = self.spectrum_func_composite_gaussian_scaled_fixed_scatter_proportion(bin_centers, eff_array, *params) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + return chi2 + + def chi_2_Poisson_simulated_resolution_scaled(self, bin_centers, data_hist_freq, eff_array, params): + nonzero_bins_index = np.where(data_hist_freq != 0) + zero_bins_index = np.where(data_hist_freq == 0) + # expectation + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fixed_scatter_proportion(bin_centers, eff_array, *params) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + return chi2 + def reduced_chi2_Pearson_Neyman_composite(self, data_hist_freq, fit_Hz, number_of_parameters): nonzero_bins_index = np.where(data_hist_freq != 0)[0] zero_bins_index = np.where(data_hist_freq == 0)[0] @@ -2412,4 +2465,314 @@ def fit_data_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, f 'data_hist_freq': data_hist_freq, 'reduced_chi2': reduced_chi2 } + return dictionary_of_fit_results + + def make_spectrum_composite_gaussian_scaled_fixed_scatter_proportion(self, survival_prob, scale_factor, emitted_peak='shake'): + p = self.scatter_proportion + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_composite_gaussian_scaled(current_working_spectrum, scale_factor) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + relative_reconstruction_eff = np.exp(-0.643*M**0.74) + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + for i in range(0, M+1): + coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M + return current_full_spectrum + + def spectrum_func_composite_gaussian_scaled_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): + + B_field = p0[0] + amplitude = p0[1] + survival_prob = p0[2] + scale_factor = p0[3] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_composite_gaussian_scaled_fixed_scatter_proportion(survival_prob, scale_factor) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def fit_data_composite_gaussian_scaled_fixed_scatter_proportion(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + FWHM_eV_guess = 5 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + sigma_guess = 5 + gamma_guess = 3 + gaussian_portion_guess = 0.5 + scale_factor_guess = 1. + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + FWHM_eV_min = 0 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + mu_min = -FWHM_eV_max + mu_max = FWHM_eV_max + gaussian_portion_min = 1e-5 + gaussian_portion_max = 1 + scale_factor_min = 1e-5 + scale_factor_max = 500 + N = len(self.gases) + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess, scale_factor_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (scale_factor_min, scale_factor_max)] + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi_2_Poisson_composite_gaussian_scaled_reso(bins_Hz, data_hist_freq, eff_array, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + survival_prob_fit = params[2] + scale_factor_fit = params[3] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + survival_prob_fit_err = perr[2] + scale_factor_fit_err = perr[3] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_composite_gaussian_scaled_fixed_scatter_proportion(bins_Hz, eff_array, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + + if print_params == True: + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Survival probability = {:.8e}'.format(survival_prob_fit) + ' +/- {:.8e}\n'.format(survival_prob_fit_err) + output_string += '-----------------\n' + output_string += 'scale factor = {:.8e}'.format(scale_factor_fit) + ' +/- {:.8e}\n'.format(scale_factor_fit_err) + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results + + def make_spectrum_simulated_resolution_scaled_fixed_scatter_proportion(self, survival_prob, scale_factor, emitted_peak='shake'): + p = self.scatter_proportion + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_simulated_resolution_scaled(current_working_spectrum, scale_factor) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + relative_reconstruction_eff = np.exp(-0.643*M**0.74) + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + for i in range(0, M+1): + coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M + return current_full_spectrum + + def spectrum_func_simulated_resolution_scaled_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): + + B_field = p0[0] + amplitude = p0[1] + survival_prob = p0[2] + scale_factor = p0[3] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_simulated_resolution_scaled_fixed_scatter_proportion(survival_prob, scale_factor) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def fit_data_simulated_resolution_scaled_fixed_scatter_proportion(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + FWHM_eV_guess = 5 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + sigma_guess = 5 + gamma_guess = 3 + gaussian_portion_guess = 0.5 + scale_factor_guess = 1. + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + FWHM_eV_min = 0 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + mu_min = -FWHM_eV_max + mu_max = FWHM_eV_max + gaussian_portion_min = 1e-5 + gaussian_portion_max = 1 + scale_factor_min = 1e-5 + scale_factor_max = 500 + N = len(self.gases) + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess, scale_factor_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (scale_factor_min, scale_factor_max)] + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi_2_Poisson_simulated_resolution_scaled(bins_Hz, data_hist_freq, eff_array, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + survival_prob_fit = params[2] + scale_factor_fit = params[3] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + survival_prob_fit_err = perr[2] + scale_factor_fit_err = perr[3] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fixed_scatter_proportion(bins_Hz, eff_array, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + + if print_params == True: + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Survival probability = {:.8e}'.format(survival_prob_fit) + ' +/- {:.8e}\n'.format(survival_prob_fit_err) + output_string += '-----------------\n' + output_string += 'scale factor = {:.8e}'.format(scale_factor_fit) + ' +/- {:.8e}\n'.format(scale_factor_fit_err) + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } return dictionary_of_fit_results \ No newline at end of file diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 116d2ea9..3093b7bb 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -8,6 +8,7 @@ import unittest import matplotlib.pyplot as plt import ROOT as r +import os from morpho.utilities import morphologging, parser logger = morphologging.getLogger(__name__) @@ -33,7 +34,7 @@ def test_complex_lineshape(self): 'max_scatters': 20, 'fixed_scatter_proportion': True, # When fixed_scatter_proportion is True, set the scatter proportion for the gases below - 'gas_scatter_proportion': [0.75, 0.25],#0.753, 0.190, 0.018, 0.039 + 'gas_scatter_proportion': [0.75, 0.25],#0.753, 0.190, 0.018, 0.039 # 0.75, 0.25 'partially_fixed_scatter_proportion': False, 'free_gases': ["H2", "He"], 'fixed_gases': ["Ar", "Kr"], @@ -41,13 +42,13 @@ def test_complex_lineshape(self): 'fixed_survival_probability': False, # When option fixed_survival_probability is True, assign the survival probability below 'survival_prob': 15/16., # assuming total cross section for elastic scattering is 1/10 of inelastic scattering - # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution - 'resolution_function': 'composite_gaussian_pedestal_factor', + # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, + 'resolution_function': 'composite_gaussian_scaled', # specific choice of parameters in the gaussian lorentzian composite resolution function 'ratio_gamma_to_sigma': 0.8, 'gaussian_proportion': 1., # if the resolution function is composite gaussian - 'sigma_array': [5.01, 13.33, 25, 11.85], + 'sigma_array': [5.01, 13.33, 15.40, 11.85], 'A_array': [0.076, 0.341, 0.381, 0.203], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown @@ -80,7 +81,7 @@ def test_complex_lineshape(self): logger.info(results['output_string']) # plot fit with shake spectrum - plt.rcParams.update({'font.size': 20}) + plt.rcParams.update({'font.size': 15}) plt.figure(figsize=(15,9)) plt.step( results['bins_Hz']/1e9, results['data_hist_freq'], @@ -89,18 +90,13 @@ def test_complex_lineshape(self): plt.plot(results['bins_Hz']/1e9, results['fit_Hz'], label = results['output_string'], alpha = 0.7) plt.legend(loc = 'upper left', fontsize = 12) plt.xlabel('frequency GHz') - plot_title = 'fit ftc march with {} gas scattering'.format(len(complexLineShape_config['gases'])) - if complexLineShape_config['fixed_scatter_proportion'] == True: - str_gas_scatter_proportion = '' - for i in range(len(complexLineShape_config['gases'])): - str_gas_scatter_proportion += complexLineShape_config['gases'][i] - str_gas_scatter_proportion += ': ' - str_gas_scatter_proportion += str(complexLineShape_config['gas_scatter_proportion'][i]) - str_gas_scatter_proportion += ' ' - #plot_title += '\n with fixed scatter proportion \n {}'.format(str_gas_scatter_proportion) + if complexLineShape_config['resolution_function'] == 'simulated_resolution_scaled': + plot_title = 'fit ftc march with gases: {},\n scatter proportion: {},\n resolution function: {},\n file for simulated resolution data: {}'.format(complexLineShape_config['gases'], complexLineShape_config['gas_scatter_proportion'], complexLineShape_config['resolution_function'], os.path.basename(complexLineShape_config['path_to_ins_resolution_data_txt'])) + elif complexLineShape_config['resolution_function'] == 'composite_gaussian_scaled': + plot_title = 'fit ftc march with gases: {},\n scatter proportion: {},\n resolution function: {},\n sigma_array: {},\n A_array: {},\n'.format(complexLineShape_config['gases'], complexLineShape_config['gas_scatter_proportion'], complexLineShape_config['resolution_function'], complexLineShape_config['sigma_array'], complexLineShape_config['A_array']) plt.title(plot_title) plt.tight_layout() - plt.savefig('/host/plots/fit_FTC_march_with_composite_gaussian_pedestal_factor_resolution.png'.format(len(complexLineShape_config['gases']))) + plt.savefig('/host/plots/fit_FTC_march_with_simulated_ins_scaled_resolution.png'.format(len(complexLineShape_config['gases']))) if __name__ == '__main__': From 86dee6ef386cfe7ba33a05e0b4b48869abfaacc7 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Sat, 5 Dec 2020 10:25:14 -0500 Subject: [PATCH 058/199] add fitting with simulated resolution scaled and fit reconstruction efficiency parameters --- .../misc/MultiGasComplexLineShape.py | 196 +++++++++++++++++- test_analysis/Complex_line_shape_fitter.py | 8 +- 2 files changed, 196 insertions(+), 8 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 376eed76..8bf45a6f 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -75,6 +75,8 @@ def InternalConfigure(self, params): if self.resolution_function == 'composite_gaussian' or 'composite_gaussian_pedestal_factor': self.A_array = reader.read_param(params, 'A_array', [0.076, 0.341, 0.381, 0.203]) self.sigma_array = reader.read_param(params, 'sigma_array', [5.01, 13.33, 15.40, 11.85]) + if self.resolution_function == 'simulated_resolution_scaled': + self.fit_recon_eff = reader.read_param(params, 'fit_recon_eff', False) #self.elevation_factor = reader.read_param(params, 'elevation_factor', 20) # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown @@ -139,7 +141,10 @@ def InternalRun(self): elif self.resolution_function == 'composite_gaussian_scaled': self.results = self.fit_data_composite_gaussian_scaled_fixed_scatter_proportion(freq_bins, data_hist_freq) elif self.resolution_function == 'simulated_resolution_scaled': - self.results = self.fit_data_simulated_resolution_scaled_fixed_scatter_proportion(freq_bins, data_hist_freq) + if self.fit_recon_eff == False: + self.results = self.fit_data_simulated_resolution_scaled_fixed_scatter_proportion(freq_bins, data_hist_freq) + else: + self.results = self.fit_data_simulated_resolution_scaled_fit_recon_eff(freq_bins, data_hist_freq) return True @@ -545,10 +550,19 @@ def chi_2_Poisson_composite_gaussian_scaled_reso(self, bin_centers, data_hist_fr return chi2 def chi_2_Poisson_simulated_resolution_scaled(self, bin_centers, data_hist_freq, eff_array, params): - nonzero_bins_index = np.where(data_hist_freq != 0) - zero_bins_index = np.where(data_hist_freq == 0) # expectation fit_Hz = self.spectrum_func_simulated_resolution_scaled_fixed_scatter_proportion(bin_centers, eff_array, *params) + nonzero_bins_index = np.where((data_hist_freq != 0) & (fit_Hz != 0)) + zero_bins_index = np.where((data_hist_freq == 0) | (fit_Hz == 0)) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + return chi2 + + def chi_2_Poisson_simulated_resolution_scaled_fit_recon_eff(self, bin_centers, data_hist_freq, eff_array, params): + # expectation + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_recon_eff(bin_centers, eff_array, *params) + nonzero_bins_index = np.where((data_hist_freq != 0) & (fit_Hz != 0)) + zero_bins_index = np.where((data_hist_freq == 0) | (fit_Hz == 0)) chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() return chi2 @@ -2637,7 +2651,7 @@ def make_spectrum_simulated_resolution_scaled_fixed_scatter_proportion(self, sur current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - relative_reconstruction_eff = np.exp(-0.643*M**0.74) + relative_reconstruction_eff = np.exp(-0.351*M**0.546) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) @@ -2651,7 +2665,7 @@ def make_spectrum_simulated_resolution_scaled_fixed_scatter_proportion(self, sur for component, i in zip(combination, range(N)): coefficient = coefficient/factorial(component)*p[i]**component for i in range(0, M+1): - coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + coefficient = coefficient*(1-0.005569990343215976*np.exp(-0.351*i**0.546)) current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum @@ -2775,4 +2789,176 @@ def fit_data_simulated_resolution_scaled_fixed_scatter_proportion(self, freq_bin 'data_hist_freq': data_hist_freq, 'reduced_chi2': reduced_chi2 } + return dictionary_of_fit_results + + def make_spectrum_simulated_resolution_scaled_fit_recon_eff(self, survival_prob, scale_factor, recon_eff_a, recon_eff_b, recon_eff_c, emitted_peak='shake'): + p = self.scatter_proportion + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_simulated_resolution_scaled(current_working_spectrum, scale_factor) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + relative_reconstruction_eff = np.exp(-1.*recon_eff_b*M**recon_eff_c) + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + for i in range(0, M+1): + coefficient = coefficient*(1-recon_eff_a*np.exp(-1.*recon_eff_b*i**recon_eff_c)) + current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M + return current_full_spectrum + + def spectrum_func_simulated_resolution_scaled_fit_recon_eff(self, bins_Hz, eff_array, *p0): + + B_field = p0[0] + amplitude = p0[1] + survival_prob = p0[2] + scale_factor = p0[3] + recon_eff_a = p0[4] + recon_eff_b = p0[5] + recon_eff_c = p0[6] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_simulated_resolution_scaled_fit_recon_eff(survival_prob, scale_factor, recon_eff_a, recon_eff_b, recon_eff_c) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def fit_data_simulated_resolution_scaled_fit_recon_eff(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + FWHM_eV_guess = 5 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + sigma_guess = 5 + gamma_guess = 3 + gaussian_portion_guess = 0.5 + scale_factor_guess = 1. + recon_eff_parameter_guess = 0.5 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + FWHM_eV_min = 0 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + mu_min = -FWHM_eV_max + mu_max = FWHM_eV_max + gaussian_portion_min = 1e-5 + gaussian_portion_max = 1 + scale_factor_min = 1e-5 + scale_factor_max = 500 + recon_eff_parameter_min = 1e-5 + recon_eff_parameter_max = 1 + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess, scale_factor_guess, recon_eff_parameter_guess, recon_eff_parameter_guess, recon_eff_parameter_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (scale_factor_min, scale_factor_max), (recon_eff_parameter_min, recon_eff_parameter_max), (recon_eff_parameter_min, recon_eff_parameter_max), (recon_eff_parameter_min, recon_eff_parameter_max)] + # Actually do the fitting + m_binned = Minuit.from_array_func(lambda p: self.chi_2_Poisson_simulated_resolution_scaled_fit_recon_eff(bins_Hz, data_hist_freq, eff_array, p), + start = p0_guess, + limit = p0_bounds, + throw_nan = True + ) + m_binned.migrad() + params = m_binned.np_values() + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + survival_prob_fit = params[2] + scale_factor_fit = params[3] + recon_eff_a_fit = params[4] + recon_eff_b_fit = params[5] + recon_eff_c_fit = params[6] + total_counts_fit = amplitude_fit + + perr = m_binned.np_errors() + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + survival_prob_fit_err = perr[2] + scale_factor_fit_err = perr[3] + recon_eff_a_fit_err = perr[4] + recon_eff_b_fit_err = perr[5] + recon_eff_c_fit_err = perr[6] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_recon_eff(bins_Hz, eff_array, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + + if print_params == True: + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'Survival probability = {:.8e}'.format(survival_prob_fit) + ' +/- {:.8e}\n'.format(survival_prob_fit_err) + output_string += '-----------------\n' + output_string += 'scale factor = {:.8e}'.format(scale_factor_fit) + ' +/- {:.8e}\n'.format(scale_factor_fit_err) + output_string += '-----------------\n' + output_string += 'recon_eff_a = {:.8e}'.format(recon_eff_a_fit) + ' +/- {:.8e}\n'.format(recon_eff_a_fit_err) + output_string += '-----------------\n' + output_string += 'recon_eff_b = {:.8e}'.format(recon_eff_b_fit) + ' +/- {:.8e}\n'.format(recon_eff_b_fit_err) + output_string += '-----------------\n' + output_string += 'recon_eff_c = {:.8e}'.format(recon_eff_c_fit) + ' +/- {:.8e}\n'.format(recon_eff_c_fit_err) + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } return dictionary_of_fit_results \ No newline at end of file diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 3093b7bb..db308bbc 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -43,13 +43,15 @@ def test_complex_lineshape(self): # When option fixed_survival_probability is True, assign the survival probability below 'survival_prob': 15/16., # assuming total cross section for elastic scattering is 1/10 of inelastic scattering # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, - 'resolution_function': 'composite_gaussian_scaled', + 'resolution_function': 'simulated_resolution_scaled', # specific choice of parameters in the gaussian lorentzian composite resolution function 'ratio_gamma_to_sigma': 0.8, 'gaussian_proportion': 1., # if the resolution function is composite gaussian 'sigma_array': [5.01, 13.33, 15.40, 11.85], - 'A_array': [0.076, 0.341, 0.381, 0.203], + 'A_array': [0.076, 0.341, 0.381, 0.203], + #parameter for simulated resolution scaled resolution + 'fit_recon_eff': False, # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -58,7 +60,7 @@ def test_complex_lineshape(self): 'shake_spectrum_parameters_json_path': '../mermithid/misc/shake_spectrum_parameters.json', 'path_to_osc_strengths_files': '/host/', 'path_to_scatter_spectra_file': '/host/', - 'path_to_ins_resolution_data_txt': '/host/res_all_conversion_max25.txt' + 'path_to_ins_resolution_data_txt': '/host/res_all_conversion_max15.5_alltraps.txt' } b = IOCicadaProcessor("reader") From 033937d11f239f95b6e34a062ecf7e87ca8e0032 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Sun, 24 Jan 2021 18:18:55 -0500 Subject: [PATCH 059/199] test git it's been a while --- test_analysis/Complex_line_shape_fitter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index db308bbc..83c12f50 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -30,11 +30,11 @@ def test_complex_lineshape(self): } complexLineShape_config = { 'bins_choice': np.linspace(0e6, 100e6, 1000), - 'gases': ["H2", "He"], + 'gases': ["H2", "He", "Ar", "Kr"], 'max_scatters': 20, 'fixed_scatter_proportion': True, # When fixed_scatter_proportion is True, set the scatter proportion for the gases below - 'gas_scatter_proportion': [0.75, 0.25],#0.753, 0.190, 0.018, 0.039 # 0.75, 0.25 + 'gas_scatter_proportion': [0.753, 0.190, 0.018, 0.039],#0.753, 0.190, 0.018, 0.039 # 0.75, 0.25 'partially_fixed_scatter_proportion': False, 'free_gases': ["H2", "He"], 'fixed_gases': ["Ar", "Kr"], From 514305d8ab154bc6637ede8f587ece4912ffd051 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 25 Jan 2021 06:44:03 -0500 Subject: [PATCH 060/199] make recon eff param configurable --- .../misc/MultiGasComplexLineShape.py | 71 +++++++++++++------ test_analysis/Complex_line_shape_fitter.py | 3 + 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 8bf45a6f..7191dcac 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -89,6 +89,10 @@ def InternalConfigure(self, params): self.path_to_missing_track_radiation_loss_data_numpy_file = '/host/' self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all4.txt') self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') + self.recon_eff_param_a = reader.read_param(params, 'recon_eff_param_a', 0.005569990343215976) + self.recon_eff_param_b = reader.read_param(params, 'recon_eff_param_b', 0.351) + self.recon_eff_param_b = reader.read_param(params, 'recon_eff_param_c', 0.546) + if not os.path.exists(self.shake_spectrum_parameters_json_path): raise IOError('Shake spectrum path does not exist') @@ -922,6 +926,9 @@ def make_spectrum_ftc(self, survival_prob, emitted_peak='shake'): #filenames = list_files('scatter_spectra_files') p = np.zeros(len(gases)) p = self.scatter_proportion + a = self.recon_eff_param_a + b = self.recon_eff_param_b + c = self.recon_eff_param_c scatter_spectra_file_path = os.path.join(current_path, 'scatter_spectra.npy') scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) en_array = self.std_eV_array() @@ -935,7 +942,7 @@ def make_spectrum_ftc(self, survival_prob, emitted_peak='shake'): current_full_spectrum += current_working_spectrum N = len(self.gases) for M in range(1, self.max_scatters + 1): - relative_reconstruction_eff = np.exp(-0.643*M**0.74) + relative_reconstruction_eff = np.exp(-b*M**c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: entry_str = '' @@ -948,7 +955,7 @@ def make_spectrum_ftc(self, survival_prob, emitted_peak='shake'): for component, i in zip(combination, range(len(self.gases))): coefficient = coefficient/factorial(component)*p[i]**component for i in range(0, M+1): - coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + coefficient = coefficient*(1-a*np.exp(-b*i**c)) current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum @@ -1383,6 +1390,9 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion(self, survival_prob, sigma, emitted_peak='shake'): p = self.scatter_proportion + a = self.recon_eff_param_a + b = self.recon_eff_param_b + c = self.recon_eff_param_c scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) en_array = self.std_eV_array() @@ -1396,7 +1406,7 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion(self, s current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - relative_reconstruction_eff = np.exp(-0.643*M**0.74) + relative_reconstruction_eff = np.exp(-b*M**c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) @@ -1410,7 +1420,7 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion(self, s for component, i in zip(combination, range(N)): coefficient = coefficient/factorial(component)*p[i]**component for i in range(0, M+1): - coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + coefficient = coefficient*(1-a*np.exp(-b*i**c)) current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum @@ -1538,6 +1548,9 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion(self, freq_b def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(self, sigma, emitted_peak='shake'): p = self.scatter_proportion + a = self.recon_eff_param_a + b = self.recon_eff_param_b + c = self.recon_eff_param_c survival_prob = self.survival_prob scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) @@ -1552,7 +1565,7 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion_and_sur current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - relative_reconstruction_eff = np.exp(-0.643*M**0.74) + relative_reconstruction_eff = np.exp(-b*M**c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) @@ -1566,7 +1579,7 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion_and_sur for component, i in zip(combination, range(N)): coefficient = coefficient/factorial(component)*p[i]**component for i in range(0, M+1): - coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + coefficient = coefficient*(1-a*np.exp(-b*i**c)) current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum @@ -1687,6 +1700,9 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival return dictionary_of_fit_results def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability(self, scatter_proportion, sigma, emitted_peak='shake'): + a = self.recon_eff_param_a + b = self.recon_eff_param_b + c = self.recon_eff_param_c p = np.zeros(len(self.gases)) p[0:-1] = scatter_proportion p[-1] = 1 - sum(scatter_proportion) @@ -1704,7 +1720,7 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability(self, current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - relative_reconstruction_eff = np.exp(-0.643*M**0.74) + relative_reconstruction_eff = np.exp(-b*M**c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) @@ -1718,7 +1734,7 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability(self, for component, i in zip(combination, range(N)): coefficient = coefficient/factorial(component)*p[i]**component for i in range(0, M+1): - coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + coefficient = coefficient*(1-a*np.exp(-b*i**c)) current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum @@ -1863,7 +1879,7 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability_parti current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - relative_reconstruction_eff = np.exp(-0.643*M**0.74) + relative_reconstruction_eff = np.exp(-b*M**c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) @@ -1877,7 +1893,7 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability_parti for component, i in zip(combination, range(N)): coefficient = coefficient/factorial(component)*p[i]**component for i in range(0, M+1): - coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + coefficient = coefficient*(1-a*np.exp(-b*i**c)) current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum @@ -2016,6 +2032,9 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_ def make_spectrum_elevated_gaussian_fixed_scatter_proportion(self, survival_prob, sigma, elevation_factor, emitted_peak='shake'): p = self.scatter_proportion + a = self.recon_eff_param_a + b = self.recon_eff_param_b + c = self.recon_eff_param_c scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) en_array = self.std_eV_array() @@ -2029,7 +2048,7 @@ def make_spectrum_elevated_gaussian_fixed_scatter_proportion(self, survival_prob current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - relative_reconstruction_eff = np.exp(-0.643*M**0.74) + relative_reconstruction_eff = np.exp(-b*M**c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) @@ -2043,7 +2062,7 @@ def make_spectrum_elevated_gaussian_fixed_scatter_proportion(self, survival_prob for component, i in zip(combination, range(N)): coefficient = coefficient/factorial(component)*p[i]**component for i in range(0, M+1): - coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + coefficient = coefficient*(1-a*np.exp(-b*i**c)) current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum @@ -2178,6 +2197,9 @@ def fit_data_elevated_gaussian_fixed_scatter_proportion(self, freq_bins, data_hi def make_spectrum_composite_gaussian_fixed_scatter_proportion(self, survival_prob, emitted_peak='shake'): p = self.scatter_proportion + a = self.recon_eff_param_a + b = self.recon_eff_param_b + c = self.recon_eff_param_c scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) en_array = self.std_eV_array() @@ -2191,7 +2213,7 @@ def make_spectrum_composite_gaussian_fixed_scatter_proportion(self, survival_pro current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - relative_reconstruction_eff = np.exp(-0.643*M**0.74) + relative_reconstruction_eff = np.exp(-b*M**c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) @@ -2205,7 +2227,7 @@ def make_spectrum_composite_gaussian_fixed_scatter_proportion(self, survival_pro for component, i in zip(combination, range(N)): coefficient = coefficient/factorial(component)*p[i]**component for i in range(0, M+1): - coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + coefficient = coefficient*(1-a*np.exp(-b*i**c)) current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum @@ -2328,6 +2350,9 @@ def fit_data_composite_gaussian_fixed_scatter_proportion(self, freq_bins, data_h def make_spectrum_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, survival_prob, pedestal_factor, emitted_peak='shake'): p = self.scatter_proportion + a = self.recon_eff_param_a + b = self.recon_eff_param_b + c = self.recon_eff_param_c scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) en_array = self.std_eV_array() @@ -2341,7 +2366,7 @@ def make_spectrum_composite_gaussian_pedestal_factor_fixed_scatter_proportion(se current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - relative_reconstruction_eff = np.exp(-0.643*M**0.74) + relative_reconstruction_eff = np.exp(-b*M**c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) @@ -2355,7 +2380,7 @@ def make_spectrum_composite_gaussian_pedestal_factor_fixed_scatter_proportion(se for component, i in zip(combination, range(N)): coefficient = coefficient/factorial(component)*p[i]**component for i in range(0, M+1): - coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + coefficient = coefficient*(1-a*np.exp(-b*i**c)) current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum @@ -2483,6 +2508,9 @@ def fit_data_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, f def make_spectrum_composite_gaussian_scaled_fixed_scatter_proportion(self, survival_prob, scale_factor, emitted_peak='shake'): p = self.scatter_proportion + a = self.recon_eff_param_a + b = self.recon_eff_param_b + c = self.recon_eff_param_c scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) en_array = self.std_eV_array() @@ -2496,7 +2524,7 @@ def make_spectrum_composite_gaussian_scaled_fixed_scatter_proportion(self, survi current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - relative_reconstruction_eff = np.exp(-0.643*M**0.74) + relative_reconstruction_eff = np.exp(-b*M**c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) @@ -2510,7 +2538,7 @@ def make_spectrum_composite_gaussian_scaled_fixed_scatter_proportion(self, survi for component, i in zip(combination, range(N)): coefficient = coefficient/factorial(component)*p[i]**component for i in range(0, M+1): - coefficient = coefficient*(1-0.023*np.exp(-0.643*i**0.74)) + coefficient = coefficient*(1-a*np.exp(-b*i**c)) current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum @@ -2638,6 +2666,9 @@ def fit_data_composite_gaussian_scaled_fixed_scatter_proportion(self, freq_bins, def make_spectrum_simulated_resolution_scaled_fixed_scatter_proportion(self, survival_prob, scale_factor, emitted_peak='shake'): p = self.scatter_proportion + a = self.recon_eff_param_a + b = self.recon_eff_param_b + c = self.recon_eff_param_c scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) en_array = self.std_eV_array() @@ -2651,7 +2682,7 @@ def make_spectrum_simulated_resolution_scaled_fixed_scatter_proportion(self, sur current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - relative_reconstruction_eff = np.exp(-0.351*M**0.546) + relative_reconstruction_eff = np.exp(-b*M**c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) @@ -2665,7 +2696,7 @@ def make_spectrum_simulated_resolution_scaled_fixed_scatter_proportion(self, sur for component, i in zip(combination, range(N)): coefficient = coefficient/factorial(component)*p[i]**component for i in range(0, M+1): - coefficient = coefficient*(1-0.005569990343215976*np.exp(-0.351*i**0.546)) + coefficient = coefficient*(1-a*np.exp(-b*i**c)) current_full_spectrum += relative_reconstruction_eff*coefficient*current_working_spectrum*survival_prob**M return current_full_spectrum diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 83c12f50..6719cae6 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -45,6 +45,9 @@ def test_complex_lineshape(self): # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, 'resolution_function': 'simulated_resolution_scaled', # specific choice of parameters in the gaussian lorentzian composite resolution function + 'recon_eff_param_a': 0.005569990343215976, + 'recon_eff_param_b': 0.351, + 'recon_eff_param_c': 0.546, 'ratio_gamma_to_sigma': 0.8, 'gaussian_proportion': 1., # if the resolution function is composite gaussian From 4bc4e14a56d2705b449641b4f1e44cc5e282b3b6 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 27 Jan 2021 22:04:09 -0500 Subject: [PATCH 061/199] fix typo in variable name recon_eff_param_c --- mermithid/processors/misc/MultiGasComplexLineShape.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 87f97cf3..91a8393d 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -95,7 +95,7 @@ def InternalConfigure(self, params): self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') self.recon_eff_param_a = reader.read_param(params, 'recon_eff_param_a', 0.005569990343215976) self.recon_eff_param_b = reader.read_param(params, 'recon_eff_param_b', 0.351) - self.recon_eff_param_b = reader.read_param(params, 'recon_eff_param_c', 0.546) + self.recon_eff_param_c = reader.read_param(params, 'recon_eff_param_c', 0.546) if not os.path.exists(self.shake_spectrum_parameters_json_path) and self.base_shape=='shake': raise IOError('Shake spectrum path does not exist') From bf8f2a1640ce01be07cadd65a6bbddf872b86c59 Mon Sep 17 00:00:00 2001 From: Xueying-Huyan Date: Thu, 28 Jan 2021 03:57:56 -0800 Subject: [PATCH 062/199] Fixed the interface between the fake data generator and the complex lineshape. Incorprated the latest deverlopment of the complex lineshape in the fake data genarator. This is a working version that successfully ran fake data generation. Expect another version today. --- .../TritiumSpectrum/FakeDataGenerator.py | 23 ++++++++++++++++--- .../misc/MultiGasComplexLineShape.py | 5 +++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 9cc65b54..6b87df97 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -97,17 +97,24 @@ def InternalConfigure(self, params): #Scattering model parameters - self.survival_prob = reader.read_param(params, 'survival_prob', 0.77) + self.survival_prob = reader.read_param(params, 'survival_prob', 1) self.scattering_sigma = reader.read_param(params, 'scattering_sigma', 18.6) self.NScatters = reader.read_param(params, 'NScatters', 20) + self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.003, 0.013, 0.014, 0.02]}) self.gases = reader.read_param(params, 'gases', ['H2', 'He']) self.scatter_proportion = reader.read_param(params, 'scatter_proportion', []) + self.fixed_survival_probability = reader.read_param(params, 'fixed_survival_probability', True) + self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) + self.sample_ins_resolution_errors = reader.read_param(params, 'sample_ins_res_errors', False) + self.resolution_function = reader.read_param(params, 'resolution_function', '') #paths self.simplified_scattering_path = reader.read_param(params, 'simplified_scattering_path', '/host/input_data/simplified_scattering_params.txt') self.detailed_scatter_spectra_path = reader.read_param(params, 'path_to_detailed_scatter_spectra_dir', '/host') - self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all.txt') self.efficiency_path = reader.read_param(params, 'efficiency_path', '') + self.rad_loss_path = reader.read_param(params, 'rad_loss_path', '') + self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all.txt') + self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/termite/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) #options self.use_lineshape = reader.read_param(params, 'use_lineshape', True) @@ -115,6 +122,7 @@ def InternalConfigure(self, params): self.apply_efficiency = reader.read_param(params, 'apply_efficiency', False) self.return_frequency = reader.read_param(params, 'return_frequency', True) + # will be replaced with complex lineshape object if detailed lineshape is used self.complexLineShape = None @@ -151,11 +159,18 @@ def InternalConfigure(self, params): complexLineShape_config = { 'gases': self.gases, 'max_scatters': self.NScatters, + 'trap_weights': self.trap_weights, 'fixed_scatter_proportion': True, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below 'gas_scatter_proportion': self.scatter_proportion, #'gas1_scatter_proportion': self.scatter_proportion, #, 1.-self.scatter_proportion], - 'use_simulated_inst_reso': True, + 'partially_fixed_scatter_proportion': False, + 'fixed_survival_probability': self.fixed_survival_probability, + 'survival_prob': self.survival_prob, + 'use_radiation_loss': self.use_radiation_loss, + 'sample_ins_res_errors': self.sample_ins_resolution_errors, + 'resolution_function': self.resolution_function, + #-----------------continue here-------------------- 'use_combined_four_trap_inst_reso': False, # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to @@ -166,6 +181,8 @@ def InternalConfigure(self, params): 'use_combined_four_trap_inst_reso': True, 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path, 'path_to_scatter_spectra_file':self.detailed_scatter_spectra_path, + 'rad_loss_path': self.rad_loss_path, + 'path_to_four_trap_ins_resolution_data_txt': self.path_to_four_trap_ins_resolution_data_txt, } logger.info('Setting up complex lineshape object') self.complexLineShape = MultiGasComplexLineShape("complexLineShape") diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 87f97cf3..4d90c678 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -67,6 +67,7 @@ def InternalConfigure(self, params): self.survival_prob = reader.read_param(params, 'survival_prob', 1) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) self.sample_ins_resolution_errors = reader.read_param(params, 'sample_ins_res_errors', False) + #-----------------continue here-------------------- # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution self.resolution_function = reader.read_param(params, 'resolution_function', '') if self.resolution_function == 'gaussian_lorentzian_composite_resolution': @@ -95,7 +96,7 @@ def InternalConfigure(self, params): self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') self.recon_eff_param_a = reader.read_param(params, 'recon_eff_param_a', 0.005569990343215976) self.recon_eff_param_b = reader.read_param(params, 'recon_eff_param_b', 0.351) - self.recon_eff_param_b = reader.read_param(params, 'recon_eff_param_c', 0.546) + self.recon_eff_param_c = reader.read_param(params, 'recon_eff_param_c', 0.546) if not os.path.exists(self.shake_spectrum_parameters_json_path) and self.base_shape=='shake': raise IOError('Shake spectrum path does not exist') @@ -2886,6 +2887,8 @@ def fit_data_simulated_resolution_scaled_fixed_scatter_proportion(self, freq_bin 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, + 'survival_prob_fit': survival_prob_fit, + 'survival_prob_fit_err': survival_prob_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, 'data_hist_freq': data_hist_freq, From cac1723c715be0a3705f2153183aa10a2d451bdb Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Thu, 28 Jan 2021 09:07:19 -0600 Subject: [PATCH 063/199] Testing merge of simulated_ins... and multigas... branches --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 1 + mermithid/processors/misc/MultiGasComplexLineShape.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 9cc65b54..11fbdac5 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -186,6 +186,7 @@ def InternalConfigure(self, params): def InternalRun(self): + logger.info("CHANGE MADE HERE") if self.return_frequency: if self.maxf == None: ROIbound = [self.minf] diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 87f97cf3..c0d66b94 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -352,6 +352,7 @@ def generate_scatter_convolution_file(self): scatter_spectra = {} for M in range(1, self.max_scatters + 1): gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + print(1) for combination in gas_scatter_combinations: mark_first_nonzero_component = 0 entry_str = '' @@ -367,8 +368,11 @@ def generate_scatter_convolution_file(self): else: scatter_to_add = scatter_spectra_single_gas[gas_type][str(component).zfill(2)] current_full_scatter = self.normalize(signal.convolve(current_full_scatter, scatter_to_add, mode='same')) - scatter_spectra[entry_str] = current_full_scatter + scatter_spectra[entry_str] = current_full_scatter + print(17) + print(len(scatter_spectra)) np.save(os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy'), scatter_spectra) + print(18) elapsed = time.time() - t logger.info('Files generated in '+str(elapsed)+'s') return @@ -3063,4 +3067,4 @@ def fit_data_simulated_resolution_scaled_fit_recon_eff(self, freq_bins, data_his 'data_hist_freq': data_hist_freq, 'reduced_chi2': reduced_chi2 } - return dictionary_of_fit_results \ No newline at end of file + return dictionary_of_fit_results From b32b50b2e1f40025e75671d5d75909de7f73e0cd Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Thu, 28 Jan 2021 10:20:28 -0600 Subject: [PATCH 064/199] Testing changes --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 1 - mermithid/processors/misc/MultiGasComplexLineShape.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 4289c243..6b87df97 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -203,7 +203,6 @@ def InternalConfigure(self, params): def InternalRun(self): - logger.info("CHANGE MADE HERE") if self.return_frequency: if self.maxf == None: ROIbound = [self.minf] diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index ddc80b5e..b83d4402 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -353,7 +353,6 @@ def generate_scatter_convolution_file(self): scatter_spectra = {} for M in range(1, self.max_scatters + 1): gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) - print(1) for combination in gas_scatter_combinations: mark_first_nonzero_component = 0 entry_str = '' @@ -370,10 +369,8 @@ def generate_scatter_convolution_file(self): scatter_to_add = scatter_spectra_single_gas[gas_type][str(component).zfill(2)] current_full_scatter = self.normalize(signal.convolve(current_full_scatter, scatter_to_add, mode='same')) scatter_spectra[entry_str] = current_full_scatter - print(17) print(len(scatter_spectra)) np.save(os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy'), scatter_spectra) - print(18) elapsed = time.time() - t logger.info('Files generated in '+str(elapsed)+'s') return From a160e85c16a8444344e2ccb02390273ac8263fbb Mon Sep 17 00:00:00 2001 From: Xueying-Huyan Date: Thu, 28 Jan 2021 21:38:23 -0800 Subject: [PATCH 065/199] added more configurables in the fake data generator. Changed variable names to be consistent across files. --- .../TritiumSpectrum/FakeDataGenerator.py | 51 ++++++++++++------- .../misc/MultiGasComplexLineShape.py | 21 ++++++-- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 6b87df97..ea6b2088 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -102,19 +102,31 @@ def InternalConfigure(self, params): self.NScatters = reader.read_param(params, 'NScatters', 20) self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.003, 0.013, 0.014, 0.02]}) self.gases = reader.read_param(params, 'gases', ['H2', 'He']) - self.scatter_proportion = reader.read_param(params, 'scatter_proportion', []) + self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) self.fixed_survival_probability = reader.read_param(params, 'fixed_survival_probability', True) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) self.sample_ins_resolution_errors = reader.read_param(params, 'sample_ins_res_errors', False) self.resolution_function = reader.read_param(params, 'resolution_function', '') + self.ratio_gamma_to_sigma = reader.read_param(params, 'ratio_gamma_to_sigma', 0.8) + self.gaussian_proportion = reader.read_param(params, 'gaussian_proportion', 0.8) + self.A_array = reader.read_param(params, 'A_array', [0.076, 0.341, 0.381, 0.203]) + self.sigma_array = reader.read_param(params, 'sigma_array', [5.01, 13.33, 15.40, 11.85]) + self.fit_recon_eff = reader.read_param(params, 'fit_recon_eff', False) + self.use_combined_four_trap_inst_reso = reader.read_param(params, 'use_combined_four_trap_inst_reso', False) + self.RF_ROI_MIN = reader.read_param(params, 'RF_ROI_MIN', 25850000000.0) + self.recon_eff_param_a = reader.read_param(params, 'recon_eff_param_a', 0.005569990343215976) + self.recon_eff_param_b = reader.read_param(params, 'recon_eff_param_b', 0.351) + self.recon_eff_param_c = reader.read_param(params, 'recon_eff_param_c', 0.546) #paths self.simplified_scattering_path = reader.read_param(params, 'simplified_scattering_path', '/host/input_data/simplified_scattering_params.txt') - self.detailed_scatter_spectra_path = reader.read_param(params, 'path_to_detailed_scatter_spectra_dir', '/host') self.efficiency_path = reader.read_param(params, 'efficiency_path', '') + self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') + self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') self.rad_loss_path = reader.read_param(params, 'rad_loss_path', '') self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all.txt') self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/termite/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) + self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') #options self.use_lineshape = reader.read_param(params, 'use_lineshape', True) @@ -143,18 +155,17 @@ def InternalConfigure(self, params): self.NScatters) elif self.lineshape=='detailed': # check path exists - if 'scatter_spectra_file' in self.detailed_scatter_spectra_path: - full_path = self.detailed_scatter_spectra_path - self.detailed_scatter_spectra_path, _ = os.path.split(full_path) + if 'scatter_spectra_file' in self.path_to_scatter_spectra_file: + full_path = self.path_to_scatter_spectra_file + self.path_to_scatter_spectra_file, _ = os.path.split(full_path) else: - full_path = os.path.join(self.detailed_scatter_spectra_path, 'scatter_spectra_file') + full_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra_file') - logger.info('Path to scatter_spectra_file: {}'.format(self.detailed_scatter_spectra_path)) + logger.info('Path to scatter_spectra_file: {}'.format(self.path_to_scatter_spectra_file)) # lineshape params self.SimpParams = [self.scattering_sigma*2*math.sqrt(2*math.log(2)), self.survival_prob] - # Setup and configure lineshape processor complexLineShape_config = { 'gases': self.gases, @@ -170,19 +181,25 @@ def InternalConfigure(self, params): 'use_radiation_loss': self.use_radiation_loss, 'sample_ins_res_errors': self.sample_ins_resolution_errors, 'resolution_function': self.resolution_function, - #-----------------continue here-------------------- - 'use_combined_four_trap_inst_reso': False, - # This is an important parameter which determines how finely resolved - # the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to - # be increased for larger datasets. + 'ratio_gamma_to_sigma': self.ratio_gamma_to_sigma, + 'gaussian_proportion': self.gaussian_proportion, + 'A_array': self.A_array, + 'sigma_array': self.sigma_array, + 'fit_recon_eff': self.fit_recon_eff, + # This is an important parameter which determines how finely resolved the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to be increased for larger datasets. 'num_points_in_std_array': 35838, + 'RF_ROI_MIN': self.RF_ROI_MIN, 'base_shape': 'dirac', - 'sample_ins_resolution_errors': True, - 'use_combined_four_trap_inst_reso': True, - 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path, - 'path_to_scatter_spectra_file':self.detailed_scatter_spectra_path, + 'path_to_osc_strengths_files': self.path_to_osc_strengths_files, + 'path_to_scatter_spectra_file':self.path_to_scatter_spectra_file, 'rad_loss_path': self.rad_loss_path, + 'path_to_ins_resolution_data_txt': self.path_to_ins_resolution_data_txt, + 'use_combined_four_trap_inst_reso': self.use_combined_four_trap_inst_reso, 'path_to_four_trap_ins_resolution_data_txt': self.path_to_four_trap_ins_resolution_data_txt, + 'path_to_quad_trap_eff_interp': self.path_to_quad_trap_eff_interp, + 'recon_eff_param_a': self.recon_eff_param_a, + 'recon_eff_param_b': self.recon_eff_param_b, + 'recon_eff_param_c': self.recon_eff_param_c } logger.info('Setting up complex lineshape object') self.complexLineShape = MultiGasComplexLineShape("complexLineShape") diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 4d90c678..0008c79b 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -67,7 +67,6 @@ def InternalConfigure(self, params): self.survival_prob = reader.read_param(params, 'survival_prob', 1) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) self.sample_ins_resolution_errors = reader.read_param(params, 'sample_ins_res_errors', False) - #-----------------continue here-------------------- # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution self.resolution_function = reader.read_param(params, 'resolution_function', '') if self.resolution_function == 'gaussian_lorentzian_composite_resolution': @@ -1269,8 +1268,8 @@ def fit_data_ftc_2(self, freq_bins, data_hist_freq): 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, - 'prob_parameter_fit': prob_parameter_fit, - 'prob_parameter_fit_err': prob_parameter_fit_err, + 'survival_prob_fit': prob_parameter_fit, + 'survival_prob_fit': prob_parameter_fit_err, 'scatter_proportion_fit': scatter_proportion_fit, 'scatter_proportion_fit_err': scatter_proportion_fit_err, 'amplitude_fit': amplitude_fit, @@ -1440,8 +1439,8 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, - 'prob_parameter_fit': prob_parameter_fit, - 'prob_parameter_fit_err': prob_parameter_fit_err, + 'survival_prob_fit': prob_parameter_fit, + 'survival_prob_fit_err': prob_parameter_fit_err, 'center_fit': scatter_proportion_fit, 'center_fit_err': scatter_proportion_fit_err, 'scale1_fit': scale1_fit, @@ -1607,6 +1606,8 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion(self, freq_b 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, + 'survival_prob_fit': survival_prob_fit, + 'survival_prob_fit_err': survival_prob_fit_err, 'sigma_fit': sigma_fit, 'sigma_fit_err': sigma_fit_err, 'amplitude_fit': amplitude_fit, @@ -2257,6 +2258,8 @@ def fit_data_elevated_gaussian_fixed_scatter_proportion(self, freq_bins, data_hi 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, + 'survival_prob_fit': survival_prob_fit, + 'survival_prob_fit_err': survival_prob_fit_err, 'sigma_fit': sigma_fit, 'sigma_fit_err': sigma_fit_err, 'amplitude_fit': amplitude_fit, @@ -2412,6 +2415,8 @@ def fit_data_composite_gaussian_fixed_scatter_proportion(self, freq_bins, data_h 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, + 'survival_prob_fit': survival_prob_fit, + 'survival_prob_fit_err': survival_prob_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, 'data_hist_freq': data_hist_freq, @@ -2570,6 +2575,8 @@ def fit_data_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, f 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, + 'survival_prob_fit': survival_prob_fit, + 'survival_prob_fit_err': survival_prob_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, 'data_hist_freq': data_hist_freq, @@ -2728,6 +2735,8 @@ def fit_data_composite_gaussian_scaled_fixed_scatter_proportion(self, freq_bins, 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, + 'survival_prob_fit': survival_prob_fit, + 'survival_prob_fit_err': survival_prob_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, 'data_hist_freq': data_hist_freq, @@ -3061,6 +3070,8 @@ def fit_data_simulated_resolution_scaled_fit_recon_eff(self, freq_bins, data_his 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, + 'survival_prob_fit': survival_prob_fit, + 'survival_prob_fit_err': survival_prob_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, 'data_hist_freq': data_hist_freq, From 1249f6a5c80d9193a858f0c3108ed530f227e33a Mon Sep 17 00:00:00 2001 From: Xueying-Huyan Date: Thu, 28 Jan 2021 23:10:58 -0800 Subject: [PATCH 066/199] organized configs --- .../TritiumSpectrum/FakeDataGenerator.py | 14 +++++++------- .../processors/misc/MultiGasComplexLineShape.py | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index ea6b2088..e3708ded 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -97,26 +97,26 @@ def InternalConfigure(self, params): #Scattering model parameters - self.survival_prob = reader.read_param(params, 'survival_prob', 1) - self.scattering_sigma = reader.read_param(params, 'scattering_sigma', 18.6) - self.NScatters = reader.read_param(params, 'NScatters', 20) - self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.003, 0.013, 0.014, 0.02]}) self.gases = reader.read_param(params, 'gases', ['H2', 'He']) self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) + self.NScatters = reader.read_param(params, 'NScatters', 20) + self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.003, 0.013, 0.014, 0.02]}) + self.survival_prob = reader.read_param(params, 'survival_prob', 0.7) self.fixed_survival_probability = reader.read_param(params, 'fixed_survival_probability', True) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) - self.sample_ins_resolution_errors = reader.read_param(params, 'sample_ins_res_errors', False) + self.sample_ins_resolution_errors = reader.read_param(params, 'sample_ins_res_errors', True) self.resolution_function = reader.read_param(params, 'resolution_function', '') self.ratio_gamma_to_sigma = reader.read_param(params, 'ratio_gamma_to_sigma', 0.8) self.gaussian_proportion = reader.read_param(params, 'gaussian_proportion', 0.8) self.A_array = reader.read_param(params, 'A_array', [0.076, 0.341, 0.381, 0.203]) self.sigma_array = reader.read_param(params, 'sigma_array', [5.01, 13.33, 15.40, 11.85]) - self.fit_recon_eff = reader.read_param(params, 'fit_recon_eff', False) + self.fit_recon_eff = reader.read_param(params, 'fit_recon_eff', True) self.use_combined_four_trap_inst_reso = reader.read_param(params, 'use_combined_four_trap_inst_reso', False) self.RF_ROI_MIN = reader.read_param(params, 'RF_ROI_MIN', 25850000000.0) self.recon_eff_param_a = reader.read_param(params, 'recon_eff_param_a', 0.005569990343215976) self.recon_eff_param_b = reader.read_param(params, 'recon_eff_param_b', 0.351) self.recon_eff_param_c = reader.read_param(params, 'recon_eff_param_c', 0.546) + self.scattering_sigma = reader.read_param(params, 'scattering_sigma', 18.6) #paths self.simplified_scattering_path = reader.read_param(params, 'simplified_scattering_path', '/host/input_data/simplified_scattering_params.txt') @@ -125,7 +125,7 @@ def InternalConfigure(self, params): self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') self.rad_loss_path = reader.read_param(params, 'rad_loss_path', '') self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all.txt') - self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/termite/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) + self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/host/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') #options diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 0008c79b..02dc9d95 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -88,10 +88,10 @@ def InternalConfigure(self, params): self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') - self.path_to_missing_track_radiation_loss_data_numpy_file = reader.read_param(params, 'rad_loss_path', '/termite/analysis_input/complex-lineshape-inputs') - self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_all.txt') + self.path_to_missing_track_radiation_loss_data_numpy_file = reader.read_param(params, 'rad_loss_path', '/host/analysis_input/complex-lineshape-inputs') + self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_all.txt') self.use_combined_four_trap_inst_reso = reader.read_param(params, 'use_combined_four_trap_inst_reso', False) - self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/termite/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/termite/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) + self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/host/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') self.recon_eff_param_a = reader.read_param(params, 'recon_eff_param_a', 0.005569990343215976) self.recon_eff_param_b = reader.read_param(params, 'recon_eff_param_b', 0.351) From 26b7b1d35ef98760525ecd09d6ac5de99b5cb018 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Fri, 29 Jan 2021 11:32:44 -0600 Subject: [PATCH 067/199] Testing recon eff in Stan --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 5 ++++- mermithid/processors/misc/MultiGasComplexLineShape.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 6b87df97..11f5dfce 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -101,6 +101,7 @@ def InternalConfigure(self, params): self.scattering_sigma = reader.read_param(params, 'scattering_sigma', 18.6) self.NScatters = reader.read_param(params, 'NScatters', 20) self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.003, 0.013, 0.014, 0.02]}) + self.recon_eff_params = reader.read_param(params, 'recon_eff_params', [0.005569990343215976, 0.351, 0.546]) self.gases = reader.read_param(params, 'gases', ['H2', 'He']) self.scatter_proportion = reader.read_param(params, 'scatter_proportion', []) self.fixed_survival_probability = reader.read_param(params, 'fixed_survival_probability', True) @@ -170,8 +171,10 @@ def InternalConfigure(self, params): 'use_radiation_loss': self.use_radiation_loss, 'sample_ins_res_errors': self.sample_ins_resolution_errors, 'resolution_function': self.resolution_function, + 'recon_eff_param_a': self.recon_eff_params[0], + 'recon_eff_param_b': self.recon_eff_params[1], + 'recon_eff_param_c': self.recon_eff_params[2], #-----------------continue here-------------------- - 'use_combined_four_trap_inst_reso': False, # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to # be increased for larger datasets. diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index b83d4402..996b9903 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -369,7 +369,6 @@ def generate_scatter_convolution_file(self): scatter_to_add = scatter_spectra_single_gas[gas_type][str(component).zfill(2)] current_full_scatter = self.normalize(signal.convolve(current_full_scatter, scatter_to_add, mode='same')) scatter_spectra[entry_str] = current_full_scatter - print(len(scatter_spectra)) np.save(os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy'), scatter_spectra) elapsed = time.time() - t logger.info('Files generated in '+str(elapsed)+'s') From b830e9c8a6034b96bcae7e3aa3ba9a15c112b614 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Fri, 29 Jan 2021 13:04:22 -0600 Subject: [PATCH 068/199] Removed double-definition of self.scatter_proportion in FakeDataGenerator --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 2264fe54..aefad410 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -98,7 +98,6 @@ def InternalConfigure(self, params): #Scattering model parameters self.gases = reader.read_param(params, 'gases', ['H2', 'He']) - self.scatter_proportion = reader.read_param(params, 'gas_scatter_proportion', []) self.NScatters = reader.read_param(params, 'NScatters', 20) self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.003, 0.013, 0.014, 0.02]}) self.recon_eff_params = reader.read_param(params, 'recon_eff_params', [0.005569990343215976, 0.351, 0.546]) @@ -172,9 +171,8 @@ def InternalConfigure(self, params): 'max_scatters': self.NScatters, 'trap_weights': self.trap_weights, 'fixed_scatter_proportion': True, - # When fix_scatter_proportion is True, set the scatter proportion for gas1 below + # When fix_scatter_proportion is True, set the scatter proportion for the gases below 'gas_scatter_proportion': self.scatter_proportion, - #'gas1_scatter_proportion': self.scatter_proportion, #, 1.-self.scatter_proportion], 'partially_fixed_scatter_proportion': False, 'fixed_survival_probability': self.fixed_survival_probability, 'survival_prob': self.survival_prob, From 2fc7e1728fb559773b90bbff341e91d4eec4c89b Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Fri, 5 Feb 2021 15:02:52 -0500 Subject: [PATCH 069/199] move self.shakeSpectrumClassInstance into InternalConfigure --- mermithid/processors/misc/MultiGasComplexLineShape.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 4bd5bf89..7a34d6cc 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -88,8 +88,8 @@ def InternalConfigure(self, params): self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') self.path_to_osc_strengths_files = reader.read_param(params, 'path_to_osc_strengths_files', '/host/') self.path_to_scatter_spectra_file = reader.read_param(params, 'path_to_scatter_spectra_file', '/host/') - self.path_to_missing_track_radiation_loss_data_numpy_file = reader.read_param(params, 'rad_loss_path', '/host/analysis_input/complex-lineshape-inputs') - self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_all.txt') + self.path_to_missing_track_radiation_loss_data_numpy_file = reader.read_param(params, 'rad_loss_path', '/host') + self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/res_cf15.5_all.txt') self.use_combined_four_trap_inst_reso = reader.read_param(params, 'use_combined_four_trap_inst_reso', False) self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/host/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') @@ -101,13 +101,12 @@ def InternalConfigure(self, params): raise IOError('Shake spectrum path does not exist') if not os.path.exists(self.path_to_osc_strengths_files): raise IOError('Path to osc strengths files does not exist') + # Read shake parameters from JSON file + self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) return True def InternalRun(self): - # Read shake parameters from JSON file - self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) - # number_of_events = len(self.data['StartFrequency']) # self.results = number_of_events From 7d295042cecce262d052f746f1699fdcb3a85c8f Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Mon, 8 Feb 2021 09:53:45 -0600 Subject: [PATCH 070/199] Commenting changes --- mermithid/processors/misc/MultiGasComplexLineShape.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 4bd5bf89..83a4925e 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -1,7 +1,7 @@ ''' Fits data to complex lineshape model. -Author: E. Machado, Y.-H. Sun, E. Novitski -Date: 4/8/20 +Author: E. Machado, Y.-H. Sun, E. Novitski, T. Weiss +Date: 1/30/2021 This processor takes in frequency data in binned histogram and fit the histogram with two gas scattering complex line shape model. From 236a737c5758ac1ad2b8be73e92b4dc24195fec2 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 8 Feb 2021 10:55:14 -0500 Subject: [PATCH 071/199] add a few comments above the make spectrum's --- .../misc/MultiGasComplexLineShape.py | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 7a34d6cc..0fb014fd 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -400,7 +400,7 @@ def check_existence_of_scatter_file(self, regenerate = True): gas_str += gas + '00' if gas_str not in list(test_dict.item().keys()): print('Gas species not matching, generating fresh files') - generate_scatter_convolution_files() + self.generate_scatter_convolution_files() return # Given a function evaluated on the SELA, convolves it with a gaussian @@ -517,7 +517,7 @@ def convolve_simulated_resolution_scaled(self, working_spectrum, scale_factor): def convolve_smeared_triangle(self, func_to_convolve, center, scale1, scale2, exponent, sigma): resolution_f = self.std_smeared_triangle(center, scale1, scale2, exponent, sigma) ans = signal.convolve(resolution_f, func_to_convolve, mode = 'same') - ans_normed = normalize(ans) + ans_normed = self.normalize(ans) return ans_normed def least_square(self, bin_centers, hist, params): @@ -1279,13 +1279,13 @@ def fit_data_ftc_2(self, freq_bins, data_hist_freq): return dictionary_of_fit_results def make_spectrum_smeared_triangle(self, prob_parameter, center, scale1, scale2, exponent, sigma, emitted_peak='shake'): - current_path = get_current_path() + current_path = self.get_current_path() # check_existence_of_scatter_files() #filenames = list_files('scatter_spectra_files') - p = np.zeros(len(gases)) - p = scatter_proportion + p = np.zeros(len(self.gases)) + p = self.scatter_proportion scatter_spectra = np.load('scatter_spectra_file/scatter_spectra.npy', allow_pickle = True) - en_array = std_eV_array() + en_array = self.std_eV_array() current_full_spectrum = np.zeros(len(en_array)) if emitted_peak == 'lorentzian': current_working_spectrum = self.std_lorenztian_17keV() @@ -1294,19 +1294,19 @@ def make_spectrum_smeared_triangle(self, prob_parameter, center, scale1, scale2, current_working_spectrum = self.convolve_smeared_triangle(current_working_spectrum, center, scale1, scale2, exponent, sigma) zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum - N = len(gases) - for M in range(1, max_scatters + 1): + N = len(self.gases) + for M in range(1, self.max_scatters + 1): gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) entry_str = '' - for component, gas_type in zip(combination, gases): + for component, gas_type in zip(combination, self.gases): entry_str += gas_type entry_str += str(component).zfill(2) current_working_spectrum = scatter_spectra.item()[entry_str] - current_working_spectrum = self.normalize(sp.signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) coefficient = factorial(sum(combination)) - for component, i in zip(combination, range(len(gases))): + for component, i in zip(combination, range(len(self.gases))): coefficient = coefficient/factorial(component)*p[i]**component*prob_parameter**M current_full_spectrum += coefficient*current_working_spectrum @@ -1323,15 +1323,15 @@ def spectrum_func_smeared_triangle(self, bins_Hz, *p0): exponent = p0[6] sigma = p0[7] - x_eV = Energy(bins_Hz, B_field) - en_loss_array = std_eV_array() + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() en_loss_array_min = en_loss_array[0] en_loss_array_max = en_loss_array[len(en_loss_array)-1] - en_array_rev = flip_array(-1*en_loss_array) + en_array_rev = ComplexLineShapeUtilities.flip_array(-1*en_loss_array) f = np.zeros(len(x_eV)) f_intermediate = np.zeros(len(x_eV)) - x_eV_minus_line = kr_line*1000 - x_eV + x_eV_minus_line = Constants.kr_line()*1000 - x_eV zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] @@ -1345,9 +1345,9 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print self.check_existence_of_scatter_files() bins_Hz = freq_bins + RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - bins_Hz_nonzero , data_hist_nonzero , data_hist_err = get_only_nonzero_bins(bins_Hz, data_hist_freq) + bins_Hz_nonzero , data_hist_nonzero , data_hist_err = self.get_only_nonzero_bins(bins_Hz, data_hist_freq) # Initial guesses for curve_fit - B_field_guess = central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) amplitude_guess = np.sum(data_hist_freq)/2 prob_parameter_guess = 0.5 center_guess = 0 @@ -1355,8 +1355,8 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print exponent_guess = 1 sigma_guess = 3 # Bounds for curve_fit - B_field_min = central_frequency_to_B_field(bins_Hz[0]) - B_field_max = central_frequency_to_B_field(bins_Hz[-1]) + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) amplitude_min = 1e-5 amplitude_max = np.sum(data_hist_freq)*3 prob_parameter_min = 1e-5 @@ -1364,7 +1364,7 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print center_min = -5 center_max = 5 width_min = 0 - width_max = Energy(bins_Hz[0], B_field_guess) - Energy(bins_Hz[-1], B_field_guess) + width_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) - ConversionFunctions.Energy(bins_Hz[-1], B_field_guess) exponent_min = 0.5 exponent_max = 2 @@ -1405,7 +1405,7 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print fit_Hz = spectrum_func(bins_Hz,*params) fit_keV = flip_array(fit_Hz) - bins_keV = Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = flip_array(bins_keV) reduced_chi2 = reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = len(params)) if print_params == True: @@ -1456,7 +1456,8 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results - + + #fitting with commposite gaussian lorentzian resolution function and fixed scatter fraction def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion(self, survival_prob, sigma, emitted_peak='shake'): p = self.scatter_proportion a = self.recon_eff_param_a @@ -1770,6 +1771,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival } return dictionary_of_fit_results + # fitting with composite gaussian lorentzian resolution function but scatter fraction floating def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability(self, scatter_proportion, sigma, emitted_peak='shake'): a = self.recon_eff_param_a b = self.recon_eff_param_b @@ -1933,9 +1935,12 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability(self, freq } return dictionary_of_fit_results + #fitting with composite gaussian lorentzian resolution function and the scatter fractions fixed for some gas species def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(self, scatter_proportion, sigma, emitted_peak='shake'): p = scatter_proportion + tuple([1-sum(scatter_proportion)-sum(self.scatter_proportion_for_fixed_gases)]) + tuple(self.scatter_proportion_for_fixed_gases) - logger.info(p) + a = self.recon_eff_param_a + b = self.recon_eff_param_b + c = self.recon_eff_param_c survival_prob = self.survival_prob scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) @@ -2101,6 +2106,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_ } return dictionary_of_fit_results + # fitting with elevated gaussian resolution function def make_spectrum_elevated_gaussian_fixed_scatter_proportion(self, survival_prob, sigma, elevation_factor, emitted_peak='shake'): p = self.scatter_proportion a = self.recon_eff_param_a @@ -2268,6 +2274,7 @@ def fit_data_elevated_gaussian_fixed_scatter_proportion(self, freq_bins, data_hi } return dictionary_of_fit_results + # fitting with superposition of gaussians as resolution function def make_spectrum_composite_gaussian_fixed_scatter_proportion(self, survival_prob, emitted_peak='shake'): p = self.scatter_proportion a = self.recon_eff_param_a @@ -2423,6 +2430,7 @@ def fit_data_composite_gaussian_fixed_scatter_proportion(self, freq_bins, data_h } return dictionary_of_fit_results + # fitting with composte gaussian resolution function with pedestal factor and fixed scatter fraction def make_spectrum_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, survival_prob, pedestal_factor, emitted_peak='shake'): p = self.scatter_proportion a = self.recon_eff_param_a From 0283d97c754fb7fbe20ada5f3cec9c2ff260fa22 Mon Sep 17 00:00:00 2001 From: Xueying-Huyan Date: Mon, 8 Feb 2021 21:02:54 -0800 Subject: [PATCH 072/199] added shake json file path in the fake data generator; fixed bug (extra argument) in TritiumSpectrum/FakeDataGenerator.py; changed the input of recon eff parameters in MultiGasComplexLineShape.py to a list to be consistent with the fake data genetator --- mermithid/misc/FakeTritiumDataFunctions.py | 2 +- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 2 ++ mermithid/processors/misc/MultiGasComplexLineShape.py | 7 ++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index f3c06fea..bc8d7941 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -332,7 +332,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, #Convolution of background and lineshape using scipy.signal.convolve -def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, max_energy, B_field, complexLineShape): +def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, max_energy, complexLineShape): """K is an array-like object """ energy_half_range = max(max_energy, abs(min_energy)) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 0016bc62..759d30f5 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -145,6 +145,7 @@ def InternalConfigure(self, params): self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/host/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') self.final_states_file = reader.read_param(params, 'final_states_file', '') + self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') #options self.use_lineshape = reader.read_param(params, 'use_lineshape', True) @@ -220,6 +221,7 @@ def InternalConfigure(self, params): 'use_combined_four_trap_inst_reso': self.use_combined_four_trap_inst_reso, 'path_to_four_trap_ins_resolution_data_txt': self.path_to_four_trap_ins_resolution_data_txt, 'path_to_quad_trap_eff_interp': self.path_to_quad_trap_eff_interp, + 'shake_spectrum_parameters_json_path': self.shake_spectrum_parameters_json_path } logger.info('Setting up complex lineshape object') self.complexLineShape = MultiGasComplexLineShape("complexLineShape") diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 1223bd89..ea647328 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -93,9 +93,10 @@ def InternalConfigure(self, params): self.use_combined_four_trap_inst_reso = reader.read_param(params, 'use_combined_four_trap_inst_reso', False) self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/host/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') - self.recon_eff_param_a = reader.read_param(params, 'recon_eff_param_a', 0.005569990343215976) - self.recon_eff_param_b = reader.read_param(params, 'recon_eff_param_b', 0.351) - self.recon_eff_param_c = reader.read_param(params, 'recon_eff_param_c', 0.546) + self.recon_eff_params = reader.read_param(params, 'recon_eff_params', [0.005569990343215976, 0.351, 0.546]) + self.recon_eff_param_a = self.recon_eff_params[0] + self.recon_eff_param_b = self.recon_eff_params[1] + self.recon_eff_param_c = self.recon_eff_params[2] if not os.path.exists(self.shake_spectrum_parameters_json_path) and self.base_shape=='shake': raise IOError('Shake spectrum path does not exist') From 7719ce226d629e6a088187bd15438d6fa724a58b Mon Sep 17 00:00:00 2001 From: Xueying-Huyan Date: Tue, 9 Feb 2021 03:06:48 -0800 Subject: [PATCH 073/199] fixed result dictionary keys for the fitted survival probability. Now the survival probabilities in all models are returned through key 'survival_prob_fit' and 'survival_prob_fit_err' --- mermithid/processors/misc/MultiGasComplexLineShape.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index ea647328..91001806 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -807,8 +807,8 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): 'B_field_fit_err': B_field_fit_err, 'FWHM_eV_fit': FWHM_eV_fit, 'FWHM_eV_fit_err': FWHM_eV_fit_err, - 'prob_parameter_fit': prob_parameter_fit, - 'prob_parameter_fit_err': prob_parameter_fit_err, + 'survival_prob_fit': prob_parameter_fit, + 'survival_prob_fit_err': prob_parameter_fit_err, 'scatter_proportion_fit': scatter_proportion_fit, 'scatter_proportion_fit_err': scatter_proportion_fit_err, 'amplitude_fit': amplitude_fit, @@ -967,8 +967,8 @@ def fit_data_1(self, freq_bins, data_hist_freq): 'B_field_fit_err': B_field_fit_err, 'FWHM_eV_fit': FWHM_eV_fit, 'FWHM_eV_fit_err': FWHM_eV_fit_err, - 'prob_parameter_fit': prob_parameter_fit, - 'prob_parameter_fit_err': prob_parameter_fit_err, + 'survival_prob_fit': prob_parameter_fit, + 'survival_prob_fit_err': prob_parameter_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, 'data_hist_freq': data_hist_freq, @@ -1269,7 +1269,7 @@ def fit_data_ftc_2(self, freq_bins, data_hist_freq): 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, 'survival_prob_fit': prob_parameter_fit, - 'survival_prob_fit': prob_parameter_fit_err, + 'survival_prob_fit_err': prob_parameter_fit_err, 'scatter_proportion_fit': scatter_proportion_fit, 'scatter_proportion_fit_err': scatter_proportion_fit_err, 'amplitude_fit': amplitude_fit, From fc3a1644ea6527a3109071ef6def84cb2aa16a4e Mon Sep 17 00:00:00 2001 From: Xueying-Huyan Date: Tue, 9 Feb 2021 13:45:49 -0800 Subject: [PATCH 074/199] changed comments about the options for the resolution function config --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 4 ++-- mermithid/processors/misc/MultiGasComplexLineShape.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 759d30f5..507560e4 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -1,7 +1,7 @@ ''' Generate binned or pseudo unbinned data -Author: T. Weiss, C. Claessens -Date:4/6/2020 +Author: T. Weiss, C. Claessens, X. Huyan +Date:2/9/2021 ''' from __future__ import absolute_import diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 91001806..670a46ca 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -1,7 +1,7 @@ ''' Fits data to complex lineshape model. -Author: E. Machado, Y.-H. Sun, E. Novitski, T. Weiss -Date: 1/30/2021 +Author: E. Machado, Y.-H. Sun, E. Novitski, T. Weiss, X. Huyan +Date: 2/9/2021 This processor takes in frequency data in binned histogram and fit the histogram with two gas scattering complex line shape model. @@ -67,7 +67,7 @@ def InternalConfigure(self, params): self.survival_prob = reader.read_param(params, 'survival_prob', 1) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) self.sample_ins_resolution_errors = reader.read_param(params, 'sample_ins_res_errors', False) - # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution + # configure the resolution functions: gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, and simulated_resolution_scaled self.resolution_function = reader.read_param(params, 'resolution_function', '') if self.resolution_function == 'gaussian_lorentzian_composite_resolution': self.ratio_gamma_to_sigma = reader.read_param(params, 'ratio_gamma_to_sigma', 0.8) From bf49a03614e19915fb019964f8f7691efb48a85c Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Tue, 9 Feb 2021 17:29:18 -0600 Subject: [PATCH 075/199] Removed unnecessary path config from FakeDataGenerator --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 0016bc62..1bd41be5 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -219,7 +219,6 @@ def InternalConfigure(self, params): 'path_to_ins_resolution_data_txt': self.path_to_ins_resolution_data_txt, 'use_combined_four_trap_inst_reso': self.use_combined_four_trap_inst_reso, 'path_to_four_trap_ins_resolution_data_txt': self.path_to_four_trap_ins_resolution_data_txt, - 'path_to_quad_trap_eff_interp': self.path_to_quad_trap_eff_interp, } logger.info('Setting up complex lineshape object') self.complexLineShape = MultiGasComplexLineShape("complexLineShape") From f16a5e64224108129cd1335690055b6de9e6f04f Mon Sep 17 00:00:00 2001 From: Xueying-Huyan Date: Tue, 9 Feb 2021 16:16:21 -0800 Subject: [PATCH 076/199] fix bug (incorrect function name) --- mermithid/processors/misc/MultiGasComplexLineShape.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 670a46ca..b1a73038 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -557,7 +557,7 @@ def chi_2_Poisson_composite_gaussian_lorentzian_reso(self, bin_centers, data_his zero_bins_index = np.where(data_hist_freq == 0) # expectation if self.fixed_scatter_proportion == True and self.fixed_survival_probability == True: - fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_survival_probability(bin_centers, eff_array, *params) + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(bin_centers, eff_array, *params) elif self.fixed_scatter_proportion == True and self.fixed_survival_probability == False: fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion(bin_centers, eff_array, *params) elif self.fixed_scatter_proportion == False and self.fixed_survival_probability == True and self.partially_fixed_scatter_proportion == False: From a0f279e887600d806814479de40bfdb6f793171e Mon Sep 17 00:00:00 2001 From: Xueying-Huyan Date: Tue, 9 Feb 2021 16:28:28 -0800 Subject: [PATCH 077/199] removed self.path_to_quad_trap_eff_interp variable (no longer needed) --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 3ce86e8f..ebb5365c 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -143,7 +143,6 @@ def InternalConfigure(self, params): self.rad_loss_path = reader.read_param(params, 'rad_loss_path', '') self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/ins_resolution_all.txt') self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/host/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) - self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') self.final_states_file = reader.read_param(params, 'final_states_file', '') self.shake_spectrum_parameters_json_path = reader.read_param(params, 'shake_spectrum_parameters_json_path', 'shake_spectrum_parameters.json') From fbacdb7ffd274449f97d061a6c3813f061019a41 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Tue, 9 Feb 2021 21:29:36 -0500 Subject: [PATCH 078/199] fix make spectrum function call bug in method spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob() --- .../misc/MultiGasComplexLineShape.py | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index b1a73038..227edd96 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -1,7 +1,7 @@ ''' Fits data to complex lineshape model. -Author: E. Machado, Y.-H. Sun, E. Novitski, T. Weiss, X. Huyan -Date: 2/9/2021 +Author: E. Machado, Y.-H. Sun, E. Novitski +Date: 4/8/20 This processor takes in frequency data in binned histogram and fit the histogram with two gas scattering complex line shape model. @@ -67,7 +67,7 @@ def InternalConfigure(self, params): self.survival_prob = reader.read_param(params, 'survival_prob', 1) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) self.sample_ins_resolution_errors = reader.read_param(params, 'sample_ins_res_errors', False) - # configure the resolution functions: gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, and simulated_resolution_scaled + # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution self.resolution_function = reader.read_param(params, 'resolution_function', '') if self.resolution_function == 'gaussian_lorentzian_composite_resolution': self.ratio_gamma_to_sigma = reader.read_param(params, 'ratio_gamma_to_sigma', 0.8) @@ -93,10 +93,9 @@ def InternalConfigure(self, params): self.use_combined_four_trap_inst_reso = reader.read_param(params, 'use_combined_four_trap_inst_reso', False) self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/host/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') - self.recon_eff_params = reader.read_param(params, 'recon_eff_params', [0.005569990343215976, 0.351, 0.546]) - self.recon_eff_param_a = self.recon_eff_params[0] - self.recon_eff_param_b = self.recon_eff_params[1] - self.recon_eff_param_c = self.recon_eff_params[2] + self.recon_eff_param_a = reader.read_param(params, 'recon_eff_param_a', 0.005569990343215976) + self.recon_eff_param_b = reader.read_param(params, 'recon_eff_param_b', 0.351) + self.recon_eff_param_c = reader.read_param(params, 'recon_eff_param_c', 0.546) if not os.path.exists(self.shake_spectrum_parameters_json_path) and self.base_shape=='shake': raise IOError('Shake spectrum path does not exist') @@ -557,7 +556,7 @@ def chi_2_Poisson_composite_gaussian_lorentzian_reso(self, bin_centers, data_his zero_bins_index = np.where(data_hist_freq == 0) # expectation if self.fixed_scatter_proportion == True and self.fixed_survival_probability == True: - fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(bin_centers, eff_array, *params) + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_survival_probability(bin_centers, eff_array, *params) elif self.fixed_scatter_proportion == True and self.fixed_survival_probability == False: fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion(bin_centers, eff_array, *params) elif self.fixed_scatter_proportion == False and self.fixed_survival_probability == True and self.partially_fixed_scatter_proportion == False: @@ -807,8 +806,8 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): 'B_field_fit_err': B_field_fit_err, 'FWHM_eV_fit': FWHM_eV_fit, 'FWHM_eV_fit_err': FWHM_eV_fit_err, - 'survival_prob_fit': prob_parameter_fit, - 'survival_prob_fit_err': prob_parameter_fit_err, + 'prob_parameter_fit': prob_parameter_fit, + 'prob_parameter_fit_err': prob_parameter_fit_err, 'scatter_proportion_fit': scatter_proportion_fit, 'scatter_proportion_fit_err': scatter_proportion_fit_err, 'amplitude_fit': amplitude_fit, @@ -967,8 +966,8 @@ def fit_data_1(self, freq_bins, data_hist_freq): 'B_field_fit_err': B_field_fit_err, 'FWHM_eV_fit': FWHM_eV_fit, 'FWHM_eV_fit_err': FWHM_eV_fit_err, - 'survival_prob_fit': prob_parameter_fit, - 'survival_prob_fit_err': prob_parameter_fit_err, + 'prob_parameter_fit': prob_parameter_fit, + 'prob_parameter_fit_err': prob_parameter_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, 'data_hist_freq': data_hist_freq, @@ -1269,7 +1268,7 @@ def fit_data_ftc_2(self, freq_bins, data_hist_freq): 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, 'survival_prob_fit': prob_parameter_fit, - 'survival_prob_fit_err': prob_parameter_fit_err, + 'survival_prob_fit': prob_parameter_fit_err, 'scatter_proportion_fit': scatter_proportion_fit, 'scatter_proportion_fit_err': scatter_proportion_fit_err, 'amplitude_fit': amplitude_fit, @@ -1673,7 +1672,7 @@ def spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_sur zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - full_spectrum = self.make_spectrum_composite_gaussian_lorentzian(sigma) + full_spectrum = self.make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(sigma) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f_intermediate = f_intermediate*eff_array f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) @@ -2591,7 +2590,7 @@ def fit_data_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, f 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results - + def make_spectrum_composite_gaussian_scaled_fixed_scatter_proportion(self, survival_prob, scale_factor, emitted_peak='shake'): p = self.scatter_proportion a = self.recon_eff_param_a From 021ddcac127a4ebe953b89228a7e3e5d1c826bb0 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Tue, 9 Feb 2021 21:33:54 -0500 Subject: [PATCH 079/199] make self.shakeSpectrumClassInstance configuration only happen when self.base_shape == 'shake' --- mermithid/processors/misc/MultiGasComplexLineShape.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 227edd96..78fec625 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -102,7 +102,8 @@ def InternalConfigure(self, params): if not os.path.exists(self.path_to_osc_strengths_files): raise IOError('Path to osc strengths files does not exist') # Read shake parameters from JSON file - self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) + if self.base_shape == 'shake': + self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) return True def InternalRun(self): From 5be7d8a2f0d2d98aac3235cb98f9b3280cf6b920 Mon Sep 17 00:00:00 2001 From: Xueying-Huyan Date: Tue, 9 Feb 2021 18:47:25 -0800 Subject: [PATCH 080/199] fixed function call in complex lineshape --- .../misc/MultiGasComplexLineShape.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 78fec625..fd470fc3 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -1,7 +1,7 @@ ''' Fits data to complex lineshape model. -Author: E. Machado, Y.-H. Sun, E. Novitski -Date: 4/8/20 +Author: E. Machado, Y.-H. Sun, E. Novitski, T. Weiss, X. Huyan +Date: 2/9/2021 This processor takes in frequency data in binned histogram and fit the histogram with two gas scattering complex line shape model. @@ -67,7 +67,7 @@ def InternalConfigure(self, params): self.survival_prob = reader.read_param(params, 'survival_prob', 1) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) self.sample_ins_resolution_errors = reader.read_param(params, 'sample_ins_res_errors', False) - # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution + # configure the resolution functions: gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, and simulated_resolution_scaled self.resolution_function = reader.read_param(params, 'resolution_function', '') if self.resolution_function == 'gaussian_lorentzian_composite_resolution': self.ratio_gamma_to_sigma = reader.read_param(params, 'ratio_gamma_to_sigma', 0.8) @@ -93,17 +93,17 @@ def InternalConfigure(self, params): self.use_combined_four_trap_inst_reso = reader.read_param(params, 'use_combined_four_trap_inst_reso', False) self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/host/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') - self.recon_eff_param_a = reader.read_param(params, 'recon_eff_param_a', 0.005569990343215976) - self.recon_eff_param_b = reader.read_param(params, 'recon_eff_param_b', 0.351) - self.recon_eff_param_c = reader.read_param(params, 'recon_eff_param_c', 0.546) + self.recon_eff_params = reader.read_param(params, 'recon_eff_params', [0.005569990343215976, 0.351, 0.546]) + self.recon_eff_param_a = self.recon_eff_params[0] + self.recon_eff_param_b = self.recon_eff_params[1] + self.recon_eff_param_c = self.recon_eff_params[2] if not os.path.exists(self.shake_spectrum_parameters_json_path) and self.base_shape=='shake': raise IOError('Shake spectrum path does not exist') if not os.path.exists(self.path_to_osc_strengths_files): raise IOError('Path to osc strengths files does not exist') # Read shake parameters from JSON file - if self.base_shape == 'shake': - self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) + self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) return True def InternalRun(self): @@ -557,7 +557,7 @@ def chi_2_Poisson_composite_gaussian_lorentzian_reso(self, bin_centers, data_his zero_bins_index = np.where(data_hist_freq == 0) # expectation if self.fixed_scatter_proportion == True and self.fixed_survival_probability == True: - fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_survival_probability(bin_centers, eff_array, *params) + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(bin_centers, eff_array, *params) elif self.fixed_scatter_proportion == True and self.fixed_survival_probability == False: fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion(bin_centers, eff_array, *params) elif self.fixed_scatter_proportion == False and self.fixed_survival_probability == True and self.partially_fixed_scatter_proportion == False: @@ -807,8 +807,8 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): 'B_field_fit_err': B_field_fit_err, 'FWHM_eV_fit': FWHM_eV_fit, 'FWHM_eV_fit_err': FWHM_eV_fit_err, - 'prob_parameter_fit': prob_parameter_fit, - 'prob_parameter_fit_err': prob_parameter_fit_err, + 'survival_prob_fit': prob_parameter_fit, + 'survival_prob_fit_err': prob_parameter_fit_err, 'scatter_proportion_fit': scatter_proportion_fit, 'scatter_proportion_fit_err': scatter_proportion_fit_err, 'amplitude_fit': amplitude_fit, @@ -967,8 +967,8 @@ def fit_data_1(self, freq_bins, data_hist_freq): 'B_field_fit_err': B_field_fit_err, 'FWHM_eV_fit': FWHM_eV_fit, 'FWHM_eV_fit_err': FWHM_eV_fit_err, - 'prob_parameter_fit': prob_parameter_fit, - 'prob_parameter_fit_err': prob_parameter_fit_err, + 'survival_prob_fit': prob_parameter_fit, + 'survival_prob_fit_err': prob_parameter_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, 'data_hist_freq': data_hist_freq, @@ -1269,7 +1269,7 @@ def fit_data_ftc_2(self, freq_bins, data_hist_freq): 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, 'survival_prob_fit': prob_parameter_fit, - 'survival_prob_fit': prob_parameter_fit_err, + 'survival_prob_fit_err': prob_parameter_fit_err, 'scatter_proportion_fit': scatter_proportion_fit, 'scatter_proportion_fit_err': scatter_proportion_fit_err, 'amplitude_fit': amplitude_fit, @@ -1673,7 +1673,7 @@ def spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_sur zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - full_spectrum = self.make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(sigma) + full_spectrum = make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(sigma) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f_intermediate = f_intermediate*eff_array f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) @@ -2591,7 +2591,7 @@ def fit_data_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, f 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results - + def make_spectrum_composite_gaussian_scaled_fixed_scatter_proportion(self, survival_prob, scale_factor, emitted_peak='shake'): p = self.scatter_proportion a = self.recon_eff_param_a From 9d6ca2c6cc970a5e0a3644ec5409fc99c7c9c19a Mon Sep 17 00:00:00 2001 From: Xueying-Huyan Date: Tue, 9 Feb 2021 18:49:48 -0800 Subject: [PATCH 081/199] included Yu-Hao's change to the shape spectrum input option --- mermithid/processors/misc/MultiGasComplexLineShape.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index fd470fc3..a1746f7b 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -103,7 +103,8 @@ def InternalConfigure(self, params): if not os.path.exists(self.path_to_osc_strengths_files): raise IOError('Path to osc strengths files does not exist') # Read shake parameters from JSON file - self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) + if self.base_shape == 'shake': + self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) return True def InternalRun(self): From a4824150551423621b62adf1a3efa6e7d3397872 Mon Sep 17 00:00:00 2001 From: Xueying-Huyan Date: Tue, 9 Feb 2021 18:53:37 -0800 Subject: [PATCH 082/199] fixed function call --- mermithid/processors/misc/MultiGasComplexLineShape.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index a1746f7b..d93ac0a5 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -1674,7 +1674,7 @@ def spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_sur zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - full_spectrum = make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(sigma) + full_spectrum = self.make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(sigma) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f_intermediate = f_intermediate*eff_array f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) From 41bbe00c6129646c2c7f63847738f64f9dffb289 Mon Sep 17 00:00:00 2001 From: Xueying-Huyan Date: Tue, 9 Feb 2021 20:05:43 -0800 Subject: [PATCH 083/199] fixed function call --- mermithid/processors/misc/MultiGasComplexLineShape.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index d93ac0a5..50b86662 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -1737,7 +1737,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival sigma_fit_err = perr[2] total_counts_fit_err = amplitude_fit_err - fit_Hz = self.spectrum_func_composite_gaussian_lorentzian(bins_Hz, eff_array, *params) + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) From 297eb80066333e0c6581e635a695a3640184d865 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Fri, 12 Feb 2021 11:09:20 -0600 Subject: [PATCH 084/199] Cleaned up code per Christine's recommendations on pull request --- mermithid/misc/FakeTritiumDataFunctions.py | 30 +------------------ .../TritiumSpectrum/FakeDataGenerator.py | 27 +++++++---------- 2 files changed, 11 insertions(+), 46 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index bc8d7941..25124ab8 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -20,8 +20,6 @@ from mermithid.misc.Constants import * from mermithid.misc.ConversionFunctions import * -import matplotlib.pyplot as plt - """ Constants and functions used by processors/TritiumSpectrum/FakeDataGenerator.py """ @@ -295,36 +293,14 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - #bins_Hz = Frequency(K_lineshape, B_field) - #lineshape_rates = complexLineShape.spectrum_func_ftc(bins_Hz, B_field, 1, ls_params[1]) - #lineshape_rates = complexLineShape.make_spectrum_1(ls_params[0]*2*np.sqrt(2*np.log(2)), ls_params[1], emitted_peak='dirac') lineshape_rates = complexLineShape.make_spectrum_ftc(ls_params[1], emitted_peak='dirac') - #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., 0., 1., ls_params[1]) lineshape_rates = np.flipud(lineshape_rates) - - fig = plt.figure() - plt.plot(K_lineshape, lineshape_rates) - plt.xlabel('Energy shift (eV)') - plt.ylabel('Complex lineshape rate') - plt.savefig('complex_lineshape_rates.pdf') - - - beta_rates = spectral_rate(K, Q, mnu, final_state_array) #np.zeros(len(K)) - #for i,ke in enumerate(K): - # beta_rates[i] = spectral_rate(ke, Q, mnu, final_state_array) + beta_rates = spectral_rate(K, Q, mnu, final_state_array) #Convolving convolved = convolve(beta_rates, lineshape_rates, mode='same') - - fig = plt.figure() - plt.plot(K, convolved) - plt.xlabel('Energy (eV)') - plt.ylabel('Signal rate') - plt.savefig('spectrum_signal.pdf') - - below_Kmin = np.where(K < Kmin) np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) return convolved @@ -348,11 +324,7 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - #bins_Hz = Frequency(K_lineshape, B_field) - #lineshape_rates = complexLineShape.spectrum_func_ftc(bins_Hz, B_field, 1, ls_params[1]) - #lineshape_rates = complexLineShape.make_spectrum_1(ls_params[0]*2*np.sqrt(2*np.log(2)), ls_params[1], emitted_peak='dirac') lineshape_rates = complexLineShape.make_spectrum_ftc(ls_params[1], emitted_peak='dirac') - #lineshape_rates = complexLineShape.spectrum_func_1(K_lineshape/1000., 0., 1., ls_params[1]) lineshape_rates = np.flipud(lineshape_rates) bkgd_rates = np.full(len(K), bkgd_rate()) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index ebb5365c..2829a4ee 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -1,7 +1,8 @@ ''' Generate binned or pseudo unbinned data Author: T. Weiss, C. Claessens, X. Huyan -Date:2/9/2021 +Date: 4/6/2020 +Updated: 2/9/2021 ''' from __future__ import absolute_import @@ -17,7 +18,6 @@ from morpho.processors import BaseProcessor from mermithid.misc.FakeTritiumDataFunctions import * from mermithid.processors.misc.MultiGasComplexLineShape import MultiGasComplexLineShape -#from mermithid.processors.misc.KrComplexLineShape import KrComplexLineShape from mermithid.misc import Constants, ComplexLineShapeUtilities, ConversionFunctions logger = morphologging.getLogger(__name__) @@ -356,18 +356,6 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 #Options of kinetic energies to be sampled self.Koptions = np.arange(Kmin_eff, Kmax_eff, step_size) - """ - def K_ls_complex(): - dE = self.Koptions[1] - self.Koptions[0] - energy_half_range = max(max_energy, abs(min_energy)) - n_dE_pos = round(energy_half_range/dE) #Number of steps for the lineshape for energies > 0 - n_dE_neg = round(energy_half_range/dE) #Same, for energies < 0 - K_lineshape = np.arange(-n_dE_neg*dE, n_dE_pos*dE, dE) - return K_lineshape - - self.complexLineShape.std_eV_array = K_ls_complex - """ - if efficiency_dict is not None: logger.info('Evaluating efficiencies') efficiency_mean, efficiency_error = efficiency_from_interpolation(self.Koptions, efficiency_dict, B_field) @@ -383,9 +371,13 @@ def K_ls_complex(): time0 = time.time() if array_method == True: - ratesS = convolved_spectral_rate_arrays(self.Koptions, Q_mean, mass, Kmin, lineshape, params, min_energy, max_energy, self.complexLineShape, self.final_state_array) + ratesS = convolved_spectral_rate_arrays(self.Koptions, Q_mean, + mass, Kmin, lineshape, params, min_energy, max_energy, + self.complexLineShape, self.final_state_array) else: - ratesS = [convolved_spectral_rate(K, Q_mean, mass, Kmin, lineshape, params, min_energy, max_energy) for K in self.Koptions] + ratesS = [convolved_spectral_rate(K, Q_mean, mass, Kmin, + lineshape, params, min_energy, max_energy) for K in + self.Koptions] # multiply rates by efficiency ratesS = ratesS*efficiency @@ -399,7 +391,8 @@ def K_ls_complex(): lineshape, params, min_energy, max_energy, self.complexLineShape) else: - ratesB = [convolved_bkgd_rate(K, Kmin, Kmax, lineshape, params, min_energy, max_energy) for K in self.Koptions] + ratesB = [convolved_bkgd_rate(K, Kmin, Kmax, lineshape, params, + min_energy, max_energy) for K in self.Koptions] time2 = time.time() logger.info('... background rate took {} s'.format(time2 - time1)) From 250d3e72709da886d88d07584c941878c861ec6c Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 15 Mar 2021 11:19:14 -0400 Subject: [PATCH 085/199] fit with modified exponential scatter peak amplitude ratios --- .../misc/MultiGasComplexLineShape.py | 180 ++++++++++++++++++ test_analysis/Complex_line_shape_fitter.py | 28 +-- 2 files changed, 196 insertions(+), 12 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 50b86662..78836bbd 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -79,6 +79,9 @@ def InternalConfigure(self, params): self.sigma_array = reader.read_param(params, 'sigma_array', [5.01, 13.33, 15.40, 11.85]) if self.resolution_function == 'simulated_resolution_scaled': self.fit_recon_eff = reader.read_param(params, 'fit_recon_eff', False) + if self.resolution_function == 'simulated_resolution_scaled_fit_scatter_peak_ratio': + self.fixed_parameter_names = reader.read_param(params, 'fixed_parameter_names', []) + self.fixed_parameter_values = reader.read_param(params, 'fixed_parameter_values', []) #self.elevation_factor = reader.read_param(params, 'elevation_factor', 20) # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown @@ -153,6 +156,8 @@ def InternalRun(self): self.results = self.fit_data_simulated_resolution_scaled_fixed_scatter_proportion(freq_bins, data_hist_freq) else: self.results = self.fit_data_simulated_resolution_scaled_fit_recon_eff(freq_bins, data_hist_freq) + elif self.resolution_function == 'simulated_resolution_scaled_fit_scatter_peak_ratio': + self.results = self.fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(freq_bins, data_hist_freq) return True @@ -3087,3 +3092,178 @@ def fit_data_simulated_resolution_scaled_fit_recon_eff(self, freq_bins, data_his 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results + + def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale_factor, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='shake'): + p = np.zeros(len(self.gases)) + p[0:-1] = scatter_fraction + p[-1] = 1 - sum(scatter_fraction) + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + current_working_spectrum = self.convolve_simulated_resolution_scaled(current_working_spectrum, scale_factor) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_b*M**scatter_peak_ratio_c) + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + current_full_spectrum += coefficient*current_working_spectrum*scatter_peak_ratio + return current_full_spectrum + + def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(self, bins_Hz, eff_array, *p0): + + B_field = p0[0] + amplitude = p0[1] + scale_factor = p0[2] + scatter_peak_ratio_b = p0[3] + scatter_peak_ratio_c = p0[4] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale_factor, scatter_peak_ratio_b, scatter_peak_ratio_c) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def chi_2_simulated_resolution_scaled_fit_scatter_peak_ratio(self, bin_centers, data_hist_freq, eff_array, params): + # expectation + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(bin_centers, eff_array, *params) + nonzero_bins_index = np.where((data_hist_freq != 0) & (fit_Hz > 0)) + zero_bins_index = np.where((data_hist_freq == 0) | (fit_Hz <= 0)) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + return chi2 + + def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + FWHM_eV_guess = 5 + prob_parameter_guess = 0.5 + scatter_proportion_guess = 0.5 + sigma_guess = 5 + gamma_guess = 3 + gaussian_portion_guess = 0.5 + scale_factor_guess = 1. + scatter_peak_ratio_parameter_guess = 0.5 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + FWHM_eV_min = 0 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) + prob_parameter_min = 1e-5 + prob_parameter_max = 1 + scatter_proportion_min = 1e-5 + scatter_proportion_max = 1 + scale_factor_min = 1e-5 + scale_factor_max = 5 + scatter_peak_ratio_parameter_min = 1e-5 + scatter_peak_ratio_parameter_max = 5 + p0_guess = [B_field_guess, amplitude_guess, scale_factor_guess, scatter_peak_ratio_parameter_guess, scatter_peak_ratio_parameter_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (scale_factor_min, scale_factor_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max)] + # Actually do the fitting + m_binned = Minuit(lambda p: self.chi_2_simulated_resolution_scaled_fit_scatter_peak_ratio(bins_Hz, data_hist_freq, eff_array, p), p0_guess, name = ['B field','amplitude','width scale factor','scatter peak ratio param b', 'scatter peak ratio param c']) + m_binned.limits = p0_bounds + if len(self.fixed_parameter_names)>0: + for fixed_parameter_name, fixed_parameter_value in zip(self.fixed_parameter_names, self.fixed_parameter_values): + m_binned.fixed[fixed_parameter_name] = True + m_binned.values[fixed_parameter_name] = fixed_parameter_value + m_binned.errors[fixed_parameter_name] = 0 + m_binned.migrad() + m_binned.hesse() + params = m_binned.values + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + scale_factor_fit = params[2] + scatter_peak_ratio_b_fit = params[3] + scatter_peak_ratio_c_fit = params[4] + total_counts_fit = amplitude_fit + + perr = m_binned.errors + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + scale_factor_fit_err = perr[2] + scatter_peak_ratio_b_fit_err = perr[3] + scatter_peak_ratio_c_fit_err = perr[4] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(bins_Hz, eff_array, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + + if print_params == True: + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'width scaling factor = {:.8e}'.format(scale_factor_fit) + ' +/- {:.8e}\n'.format(scale_factor_fit_err) + output_string += '-----------------\n' + output_string += 'scatter_peak_ratio_b = {:.8e}'.format(scatter_peak_ratio_b_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_b_fit_err) + output_string += '-----------------\n' + output_string += 'scatter_peak_ratio_c = {:.8e}'.format(scatter_peak_ratio_c_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_c_fit_err) + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'scale_factor_fit': scale_factor_fit, + 'scale_factor_fit_err': scale_factor_fit_err, + 'scatter_peak_ratio_b_fit': scatter_peak_ratio_b_fit, + 'scatter_peak_ratio_b_fit_err': scatter_peak_ratio_b_fit_err, + 'scatter_peak_ratio_c_fit': scatter_peak_ratio_c_fit, + 'scatter_peak_ratio_c_fit_err': scatter_peak_ratio_c_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + return dictionary_of_fit_results \ No newline at end of file diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 6719cae6..ef36b7d0 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -28,13 +28,14 @@ def test_complex_lineshape(self): "use_katydid": False, "variables": ['StartTimeInAcq','StartFrequency'] } + complexLineShape_config = { 'bins_choice': np.linspace(0e6, 100e6, 1000), - 'gases': ["H2", "He", "Ar", "Kr"], + 'gases': ["H2", "He"], # Ar, Kr 'max_scatters': 20, 'fixed_scatter_proportion': True, # When fixed_scatter_proportion is True, set the scatter proportion for the gases below - 'gas_scatter_proportion': [0.753, 0.190, 0.018, 0.039],#0.753, 0.190, 0.018, 0.039 # 0.75, 0.25 + 'gas_scatter_proportion': [0.8, 0.2],#0.827, 0.076, 0.068, 0.028 # 0.75, 0.25 'partially_fixed_scatter_proportion': False, 'free_gases': ["H2", "He"], 'fixed_gases': ["Ar", "Kr"], @@ -42,8 +43,8 @@ def test_complex_lineshape(self): 'fixed_survival_probability': False, # When option fixed_survival_probability is True, assign the survival probability below 'survival_prob': 15/16., # assuming total cross section for elastic scattering is 1/10 of inelastic scattering - # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, - 'resolution_function': 'simulated_resolution_scaled', + # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, 'simulated_resolution_scaled_fit_scatter_peak_ratio' + 'resolution_function': 'simulated_resolution_scaled_fit_scatter_peak_ratio', # specific choice of parameters in the gaussian lorentzian composite resolution function 'recon_eff_param_a': 0.005569990343215976, 'recon_eff_param_b': 0.351, @@ -54,7 +55,11 @@ def test_complex_lineshape(self): 'sigma_array': [5.01, 13.33, 15.40, 11.85], 'A_array': [0.076, 0.341, 0.381, 0.203], #parameter for simulated resolution scaled resolution - 'fit_recon_eff': False, + 'fit_recon_eff': False, + #parameters for simulated resolution scaled with scatter peak ratio fitted + #choose the parameters you want to fix from ['B field','amplitude','width scale factor','scatter peak ratio param b', 'scatter peak ratio param c'], + 'fixed_parameter_names': ['scatter peak ratio param c'], + 'fixed_parameter_values': [1.0], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -67,16 +72,16 @@ def test_complex_lineshape(self): } b = IOCicadaProcessor("reader") - complexLineShape = MultiGasComplexLineShape("complexLineShape") - b.Configure(reader_config) - complexLineShape.Configure(complexLineShape_config) - b.Run() data = b.data logger.info("Data extracted = {}".format(data.keys())) for key in data.keys(): logger.info("{} -> size = {}".format(key,len(data[key]))) + + complexLineShape = MultiGasComplexLineShape("complexLineShape") + + complexLineShape.Configure(complexLineShape_config) complexLineShape.data = data @@ -95,9 +100,8 @@ def test_complex_lineshape(self): plt.plot(results['bins_Hz']/1e9, results['fit_Hz'], label = results['output_string'], alpha = 0.7) plt.legend(loc = 'upper left', fontsize = 12) plt.xlabel('frequency GHz') - if complexLineShape_config['resolution_function'] == 'simulated_resolution_scaled': - plot_title = 'fit ftc march with gases: {},\n scatter proportion: {},\n resolution function: {},\n file for simulated resolution data: {}'.format(complexLineShape_config['gases'], complexLineShape_config['gas_scatter_proportion'], complexLineShape_config['resolution_function'], os.path.basename(complexLineShape_config['path_to_ins_resolution_data_txt'])) - elif complexLineShape_config['resolution_function'] == 'composite_gaussian_scaled': + plot_title = 'fit ftc march with gases: {},\n scatter proportion: {},\n resolution function: {},\n file for simulated resolution data: {}'.format(complexLineShape_config['gases'], complexLineShape_config['gas_scatter_proportion'], complexLineShape_config['resolution_function'], os.path.basename(complexLineShape_config['path_to_ins_resolution_data_txt'])) + if complexLineShape_config['resolution_function'] == 'composite_gaussian_scaled': plot_title = 'fit ftc march with gases: {},\n scatter proportion: {},\n resolution function: {},\n sigma_array: {},\n A_array: {},\n'.format(complexLineShape_config['gases'], complexLineShape_config['gas_scatter_proportion'], complexLineShape_config['resolution_function'], complexLineShape_config['sigma_array'], complexLineShape_config['A_array']) plt.title(plot_title) plt.tight_layout() From 2f6f6ed8ac4ca0040649d2ee53926f4d4a2dfdc6 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Mon, 15 Mar 2021 22:14:55 -0500 Subject: [PATCH 086/199] Added function to compute frac events in ROI for count rate predictions --- mermithid/misc/FakeTritiumDataFunctions.py | 35 ++++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 25124ab8..0e98d380 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -340,18 +340,27 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, -##Fraction of events near the endpoint -##Currently, this only holds for the last 13.6 eV of the spectrum -#def frac_near_endpt(Kmin, Q, mass, atom_or_mol='atom'): -# A = integrate.quad(spectral_rate, Kmin, Q-mass, args=(Q,mass)) -# B = integrate.quad(spectral_rate, V0, Q-mass, args=(Q,mass)) #Minimum at V0 because electrons with energy below screening barrier do not escape -# f = (A[0])/(B[0]) -# if atom_or_mol=='atom': -# return 0.7006*f -# elif atom_or_mol=='mol' or atom_or_mol=='molecule': -# return 0.57412*f -# else: -# print("Choose 'atom' or 'mol'.") +#Fraction of events near the endpoint +def frac_near_endpt(Kmin, Q, mass, final_state_array, atom_or_mol='mol', range='wide'): + """ + Options for range: + - 'narrow': Only extends ~18 eV (or less) below the endpoint, so that all decays are to the ground state + - 'wide': Wide enough that the probability of decay to a 3He electronic energy level that would shift Q below the ROI is very low + """ + A = integrate.quad(spectral_rate, Kmin, Q-mass, args=(Q, mass, final_state_array)) + B = integrate.quad(spectral_rate, V0, Q-mass, args=(Q, mass, final_state_array)) #Minimum at V0 because electrons with energy below screening barrier do not escape + f = (A[0])/(B[0]) + if range=='narrow': + if atom_or_mol=='atom': + return 0.7006*f + elif atom_or_mol=='mol' or atom_or_mol=='molecule': + return 0.57412*f + else: + logger.warn("Choose 'atom' or 'mol'.") + elif range=='wide': + return f + else: + logger.warn("Choose range 'narrow' or 'wide'") #Convert [number of particles]=(density*volume*efficiency) to a signal activity A_s, measured in events/second. @@ -359,7 +368,7 @@ def find_signal_activity(Nparticles, m, Q, Kmin, atom_or_mol='atom', nTperMolecu """ Functions to calculate number of events to generate """ - br = frac_near_endpt(Kmin, Q, m, atom_or_mol) + br = frac_near_endpt(Kmin, Q, m, final_state_array, atom_or_mol) Thalflife = 3.8789*10**8 A_s = Nparticles*np.log(2)/(Thalflife)*br if atom_or_mol=='atom': From aa3a1c3831e7b21145e215109f92d63b5139b747 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 15 Mar 2021 23:42:11 -0400 Subject: [PATCH 087/199] add survival probability in parameters --- .../misc/MultiGasComplexLineShape.py | 59 ++++++++++++------- test_analysis/Complex_line_shape_fitter.py | 8 +-- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 78836bbd..e265f650 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -3093,7 +3093,7 @@ def fit_data_simulated_resolution_scaled_fit_recon_eff(self, freq_bins, data_his } return dictionary_of_fit_results - def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale_factor, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='shake'): + def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale_factor, survival_probability, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='shake'): p = np.zeros(len(self.gases)) p[0:-1] = scatter_fraction p[-1] = 1 - sum(scatter_fraction) @@ -3123,7 +3123,7 @@ def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale coefficient = factorial(sum(combination)) for component, i in zip(combination, range(N)): coefficient = coefficient/factorial(component)*p[i]**component - current_full_spectrum += coefficient*current_working_spectrum*scatter_peak_ratio + current_full_spectrum += coefficient*current_working_spectrum*scatter_peak_ratio*survival_probability**M return current_full_spectrum def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(self, bins_Hz, eff_array, *p0): @@ -3131,8 +3131,11 @@ def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(self, bins_ B_field = p0[0] amplitude = p0[1] scale_factor = p0[2] - scatter_peak_ratio_b = p0[3] - scatter_peak_ratio_c = p0[4] + survival_probability = p0[3] + scatter_peak_ratio_b = p0[4] + scatter_peak_ratio_c = p0[5] + N = len(self.gases) + scatter_fraction = p0[6:5+N] x_eV = ConversionFunctions.Energy(bins_Hz, B_field) en_loss_array = self.std_eV_array() @@ -3145,7 +3148,7 @@ def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(self, bins_ zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - full_spectrum = self.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale_factor, scatter_peak_ratio_b, scatter_peak_ratio_c) + full_spectrum = self.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale_factor, survival_probability, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f_intermediate = f_intermediate*eff_array f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) @@ -3173,8 +3176,8 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) amplitude_guess = np.sum(data_hist_freq)/2 FWHM_eV_guess = 5 - prob_parameter_guess = 0.5 - scatter_proportion_guess = 0.5 + survival_probability_guess = 0.5 + scatter_fraction_guess = 0.5 sigma_guess = 5 gamma_guess = 3 gaussian_portion_guess = 0.5 @@ -3187,18 +3190,23 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, amplitude_max = np.sum(data_hist_freq)*3 FWHM_eV_min = 0 FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) - prob_parameter_min = 1e-5 - prob_parameter_max = 1 - scatter_proportion_min = 1e-5 - scatter_proportion_max = 1 + survival_probability_min = 1e-5 + survival_probability_max = 1 + scatter_fraction_min = 1e-5 + scatter_fraction_max = 1 scale_factor_min = 1e-5 scale_factor_max = 5 scatter_peak_ratio_parameter_min = 1e-5 scatter_peak_ratio_parameter_max = 5 - p0_guess = [B_field_guess, amplitude_guess, scale_factor_guess, scatter_peak_ratio_parameter_guess, scatter_peak_ratio_parameter_guess] - p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (scale_factor_min, scale_factor_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max)] + N = len(self.gases) + gas_scatter_fraction_parameter_str = [] + for i in range(N-1): + gas_scatter_fraction_parameter_str += [self.gases[i]+' scatter fraction'] + p0_guess = [B_field_guess, amplitude_guess, scale_factor_guess, survival_probability_guess, scatter_peak_ratio_parameter_guess, scatter_peak_ratio_parameter_guess]+ (N-1)*[scatter_fraction_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (scale_factor_min, scale_factor_max), (survival_probability_min, survival_probability_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max)] + (N-1)*[(scatter_fraction_min, scatter_fraction_max)] + parameter_names = ['B field','amplitude','width scale factor', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] + gas_scatter_fraction_parameter_str # Actually do the fitting - m_binned = Minuit(lambda p: self.chi_2_simulated_resolution_scaled_fit_scatter_peak_ratio(bins_Hz, data_hist_freq, eff_array, p), p0_guess, name = ['B field','amplitude','width scale factor','scatter peak ratio param b', 'scatter peak ratio param c']) + m_binned = Minuit(lambda p: self.chi_2_simulated_resolution_scaled_fit_scatter_peak_ratio(bins_Hz, data_hist_freq, eff_array, p), p0_guess, name = parameter_names) m_binned.limits = p0_bounds if len(self.fixed_parameter_names)>0: for fixed_parameter_name, fixed_parameter_value in zip(self.fixed_parameter_names, self.fixed_parameter_values): @@ -3207,22 +3215,27 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, m_binned.errors[fixed_parameter_name] = 0 m_binned.migrad() m_binned.hesse() - params = m_binned.values + params = m_binned.values[0:] B_field_fit = params[0] #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) amplitude_fit = params[1] scale_factor_fit = params[2] - scatter_peak_ratio_b_fit = params[3] - scatter_peak_ratio_c_fit = params[4] + survival_probability_fit = params[3] + scatter_peak_ratio_b_fit = params[4] + scatter_peak_ratio_c_fit = params[5] total_counts_fit = amplitude_fit + logger.info('\n'+str(m_binned.params)) + scatter_fraction_fit = params[6:5+N]+[1- sum(params[6:5+N])] - perr = m_binned.errors + perr = m_binned.errors[0:] B_field_fit_err = perr[0] amplitude_fit_err = perr[1] scale_factor_fit_err = perr[2] - scatter_peak_ratio_b_fit_err = perr[3] - scatter_peak_ratio_c_fit_err = perr[4] + survival_probability_fit_err = perr[3] + scatter_peak_ratio_b_fit_err = perr[4] + scatter_peak_ratio_c_fit_err = perr[5] total_counts_fit_err = amplitude_fit_err + scatter_fraction_fit_err = perr[6:5+N]+[np.sqrt(sum(np.array(perr[6:5+N])**2))] fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) @@ -3240,10 +3253,16 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, output_string += '-----------------\n' output_string += 'width scaling factor = {:.8e}'.format(scale_factor_fit) + ' +/- {:.8e}\n'.format(scale_factor_fit_err) output_string += '-----------------\n' + output_string += 'survival probability = {:.8e}'.format(survival_probability_fit) + ' +/- {:.8e}\n'.format(survival_probability_fit_err) + output_string += '-----------------\n' output_string += 'scatter_peak_ratio_b = {:.8e}'.format(scatter_peak_ratio_b_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_b_fit_err) output_string += '-----------------\n' output_string += 'scatter_peak_ratio_c = {:.8e}'.format(scatter_peak_ratio_c_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_c_fit_err) output_string += '-----------------\n' + for i in range(len(self.gases)): + output_string += '{} scatter fraction \n= '.format(self.gases[i]) + "{:.8e}".format(scatter_fraction_fit[i])\ + +' +/- ' + "{:.8e}".format(scatter_fraction_fit_err[i])+'\n' + output_string += '-----------------\n' elapsed = time.time() - t output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' dictionary_of_fit_results = { diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index ef36b7d0..e449a09e 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -57,9 +57,9 @@ def test_complex_lineshape(self): #parameter for simulated resolution scaled resolution 'fit_recon_eff': False, #parameters for simulated resolution scaled with scatter peak ratio fitted - #choose the parameters you want to fix from ['B field','amplitude','width scale factor','scatter peak ratio param b', 'scatter peak ratio param c'], - 'fixed_parameter_names': ['scatter peak ratio param c'], - 'fixed_parameter_values': [1.0], + #choose the parameters you want to fix from ['B field','amplitude','width scale factor', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] plus the gas scatter fractions as ['H2 scatter fraction'], + 'fixed_parameter_names': ['survival probability', 'H2 scatter fraction'], + 'fixed_parameter_values': [0.8, 0.9], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -100,7 +100,7 @@ def test_complex_lineshape(self): plt.plot(results['bins_Hz']/1e9, results['fit_Hz'], label = results['output_string'], alpha = 0.7) plt.legend(loc = 'upper left', fontsize = 12) plt.xlabel('frequency GHz') - plot_title = 'fit ftc march with gases: {},\n scatter proportion: {},\n resolution function: {},\n file for simulated resolution data: {}'.format(complexLineShape_config['gases'], complexLineShape_config['gas_scatter_proportion'], complexLineShape_config['resolution_function'], os.path.basename(complexLineShape_config['path_to_ins_resolution_data_txt'])) + plot_title = 'fit ftc march with gases: {},\n resolution function: {},\n file for simulated resolution data: {}'.format(complexLineShape_config['gases'], complexLineShape_config['resolution_function'], os.path.basename(complexLineShape_config['path_to_ins_resolution_data_txt'])) if complexLineShape_config['resolution_function'] == 'composite_gaussian_scaled': plot_title = 'fit ftc march with gases: {},\n scatter proportion: {},\n resolution function: {},\n sigma_array: {},\n A_array: {},\n'.format(complexLineShape_config['gases'], complexLineShape_config['gas_scatter_proportion'], complexLineShape_config['resolution_function'], complexLineShape_config['sigma_array'], complexLineShape_config['A_array']) plt.title(plot_title) From 79909fd83261459759ea12542325e5760c6e2a35 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Wed, 31 Mar 2021 18:29:10 -0500 Subject: [PATCH 088/199] Updated FakeDataGenerator to use correct recon eff model --- mermithid/misc/FakeTritiumDataFunctions.py | 8 ++++---- .../TritiumSpectrum/FakeDataGenerator.py | 16 +++++++++------- .../processors/misc/MultiGasComplexLineShape.py | 4 ++-- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 0e98d380..1132ad1a 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -274,7 +274,7 @@ def convolved_bkgd_rate(K, Kmin, Kmax, lineshape, ls_params, min_energy, max_ene #Convolution of signal and lineshape using scipy.signal.convolve def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, - lineshape, ls_params, min_energy, max_energy, + lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, min_energy, max_energy, complexLineShape, final_state_array): """K is an array-like object """ @@ -293,7 +293,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - lineshape_rates = complexLineShape.make_spectrum_ftc(ls_params[1], emitted_peak='dirac') + lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(1, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, emitted_peak='dirac') lineshape_rates = np.flipud(lineshape_rates) beta_rates = spectral_rate(K, Q, mnu, final_state_array) @@ -308,7 +308,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, #Convolution of background and lineshape using scipy.signal.convolve -def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, max_energy, complexLineShape): +def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, min_energy, max_energy, complexLineShape): """K is an array-like object """ energy_half_range = max(max_energy, abs(min_energy)) @@ -324,7 +324,7 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, min_energy, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - lineshape_rates = complexLineShape.make_spectrum_ftc(ls_params[1], emitted_peak='dirac') + lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(1, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, emitted_peak='dirac') lineshape_rates = np.flipud(lineshape_rates) bkgd_rates = np.full(len(K), bkgd_rate()) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 2829a4ee..a400abcc 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -54,7 +54,8 @@ def InternalConfigure(self, params): – gases: list of strings naming gases to be included in complex lineshape model. Options: 'H2', 'He', 'Kr', 'Ar', 'CO' - NScatters: lineshape parameter - number of scatters included in lineshape - trap_weights: distionary of two lists, labeled 'weights' and 'errors', which respectively include the fractions of counts from each trap and the uncertainties on those fractions - – recon_eff_params: list of parameters parameterizing function for reconstruction efficiency as a function of scattering peak order, in complex lineshape + - scatter_peak_ratio_b: "b" in reconstrudction efficiency curve model: e^(-b*i^c), where i is the scatter order + - scatter_peak_ratio_c: "c" in the same reconstruction efficiency model – scatter_proportion: list of proportion of scatters due to each gas in self.gases (in the same order), in complex lineshape - survival_prob: lineshape parameter - probability of electron staying in the trap between two inelastics scatters (it could escape due to elastics scatters or the inelastics scatters, themselves) – use_radiation_loss: if True, radiation loss will be included in the complex lineshape; should be set to True except for testing purposes @@ -120,7 +121,9 @@ def InternalConfigure(self, params): self.gases = reader.read_param(params, 'gases', ['H2', 'He']) self.NScatters = reader.read_param(params, 'NScatters', 20) self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.003, 0.013, 0.014, 0.02]}) - self.recon_eff_params = reader.read_param(params, 'recon_eff_params', [0.005569990343215976, 0.351, 0.546]) + #self.recon_eff_params = reader.read_param(params, 'recon_eff_params', [0.005569990343215976, 0.351, 0.546]) + self.scatter_peak_ratio_b = reader.read_param(params, 'scatter_peak_ratio_b', 0.686312493) + self.scatter_peak_ratio_c = reader.read_param(params, 'scatter_peak_ratio_c', 0.52481056) self.scatter_proportion = reader.read_param(params, 'scatter_proportion', []) self.survival_prob = reader.read_param(params, 'survival_prob', 0.7) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) @@ -199,9 +202,8 @@ def InternalConfigure(self, params): 'use_radiation_loss': self.use_radiation_loss, 'sample_ins_res_errors': self.sample_ins_resolution_errors, 'resolution_function': self.resolution_function, - 'recon_eff_param_a': self.recon_eff_params[0], - 'recon_eff_param_b': self.recon_eff_params[1], - 'recon_eff_param_c': self.recon_eff_params[2], + 'scatter_peak_ratio_b': self.scatter_peak_ratio_b, + 'scatter_peak_ratio_c': self.scatter_peak_ratio_c, 'fit_recon_eff': self.fit_recon_eff, #For analytics resolution functions, only: @@ -372,11 +374,11 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 if array_method == True: ratesS = convolved_spectral_rate_arrays(self.Koptions, Q_mean, - mass, Kmin, lineshape, params, min_energy, max_energy, + mass, Kmin, lineshape, params, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, min_energy, max_energy, self.complexLineShape, self.final_state_array) else: ratesS = [convolved_spectral_rate(K, Q_mean, mass, Kmin, - lineshape, params, min_energy, max_energy) for K in + lineshape, params, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, min_energy, max_energy) for K in self.Koptions] # multiply rates by efficiency diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index e265f650..0ad50178 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -1125,7 +1125,7 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): } return dictionary_of_fit_results - #simulated resolution with scatter_proportion floating, without reconstruction eff curve, withou detection eff curve + #simulated resolution with scatter_proportion floating, without reconstruction eff curve, without detection eff curve def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak='shake'): gases = self.gases current_path = self.path_to_scatter_spectra_file @@ -3285,4 +3285,4 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, 'data_hist_freq': data_hist_freq, 'reduced_chi2': reduced_chi2 } - return dictionary_of_fit_results \ No newline at end of file + return dictionary_of_fit_results From 0c3398c44e386236a94b0e01cb0ba588f0b63718 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Wed, 31 Mar 2021 19:24:10 -0500 Subject: [PATCH 089/199] Added Dirac delta option to new complex lineshape function, fixed errors --- mermithid/misc/FakeTritiumDataFunctions.py | 8 ++++---- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 8 ++++---- mermithid/processors/misc/MultiGasComplexLineShape.py | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 1132ad1a..e1de6b3c 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -274,7 +274,7 @@ def convolved_bkgd_rate(K, Kmin, Kmax, lineshape, ls_params, min_energy, max_ene #Convolution of signal and lineshape using scipy.signal.convolve def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, - lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, min_energy, max_energy, + lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, min_energy, max_energy, complexLineShape, final_state_array): """K is an array-like object """ @@ -293,7 +293,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(1, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, emitted_peak='dirac') + lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(1, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') lineshape_rates = np.flipud(lineshape_rates) beta_rates = spectral_rate(K, Q, mnu, final_state_array) @@ -308,7 +308,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, #Convolution of background and lineshape using scipy.signal.convolve -def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, min_energy, max_energy, complexLineShape): +def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, min_energy, max_energy, complexLineShape): """K is an array-like object """ energy_half_range = max(max_energy, abs(min_energy)) @@ -324,7 +324,7 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(1, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, emitted_peak='dirac') + lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(1, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') lineshape_rates = np.flipud(lineshape_rates) bkgd_rates = np.full(len(K), bkgd_rate()) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index a400abcc..1858f931 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -118,7 +118,7 @@ def InternalConfigure(self, params): #Scattering model parameters - self.gases = reader.read_param(params, 'gases', ['H2', 'He']) + self.gases = reader.read_param(params, 'gases', ['H2', 'He', 'CO']) self.NScatters = reader.read_param(params, 'NScatters', 20) self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.003, 0.013, 0.014, 0.02]}) #self.recon_eff_params = reader.read_param(params, 'recon_eff_params', [0.005569990343215976, 0.351, 0.546]) @@ -374,11 +374,11 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 if array_method == True: ratesS = convolved_spectral_rate_arrays(self.Koptions, Q_mean, - mass, Kmin, lineshape, params, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, min_energy, max_energy, + mass, Kmin, lineshape, params, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, self.scatter_proportion, min_energy, max_energy, self.complexLineShape, self.final_state_array) else: ratesS = [convolved_spectral_rate(K, Q_mean, mass, Kmin, - lineshape, params, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, min_energy, max_energy) for K in + lineshape, params, min_energy, max_energy) for K in self.Koptions] # multiply rates by efficiency @@ -390,7 +390,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 # background if array_method == True: ratesB = convolved_bkgd_rate_arrays(self.Koptions, Kmin, Kmax, - lineshape, params, min_energy, max_energy, + lineshape, params, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, self.scatter_proportion, min_energy, max_energy, self.complexLineShape) else: ratesB = [convolved_bkgd_rate(K, Kmin, Kmax, lineshape, params, diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 0ad50178..14ec33ac 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -3105,6 +3105,8 @@ def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + elif emitted_peak == 'dirac': + current_working_spectrum = self.std_dirac() current_working_spectrum = self.convolve_simulated_resolution_scaled(current_working_spectrum, scale_factor) zeroth_order_peak = current_working_spectrum current_full_spectrum += zeroth_order_peak From 69bcf346540d4e3630824ec5190dffb30141d1c7 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 23 Jun 2021 18:42:01 -0400 Subject: [PATCH 090/199] add fitting with gaussian resolution --- .../misc/MultiGasComplexLineShape.py | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 14ec33ac..bf57c482 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -3288,3 +3288,198 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results + + + def make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(self, gauss_FWHM_eV, survival_probability, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='shake'): + p = np.zeros(len(self.gases)) + p[0:-1] = scatter_fraction + p[-1] = 1 - sum(scatter_fraction) + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + elif emitted_peak == 'dirac': + current_working_spectrum = self.std_dirac() + current_working_spectrum = self.convolve_gaussian(current_working_spectrum, gauss_FWHM_eV) + zeroth_order_peak = current_working_spectrum + current_full_spectrum += zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_b*M**scatter_peak_ratio_c) + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + #print(combination) + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + current_full_spectrum += coefficient*current_working_spectrum*scatter_peak_ratio*survival_probability**M + return current_full_spectrum + + def spectrum_func_gaussian_resolution_fit_scatter_peak_ratio(self, bins_Hz, eff_array, *p0): + + B_field = p0[0] + amplitude = p0[1] + gauss_FWHM_eV = p0[2] + survival_probability = p0[3] + scatter_peak_ratio_b = p0[4] + scatter_peak_ratio_c = p0[5] + N = len(self.gases) + scatter_fraction = p0[6:5+N] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(gauss_FWHM_eV, survival_probability, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def chi_2_gaussian_resolution_fit_scatter_peak_ratio(self, bin_centers, data_hist_freq, eff_array, params): + # expectation + fit_Hz = self.spectrum_func_gaussian_resolution_fit_scatter_peak_ratio(bin_centers, eff_array, *params) + nonzero_bins_index = np.where((data_hist_freq != 0) & (fit_Hz > 0)) + zero_bins_index = np.where((data_hist_freq == 0) | (fit_Hz <= 0)) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + return chi2 + + def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq)/2 + gauss_FWHM_eV_guess = 1 + survival_probability_guess = 0.5 + scatter_fraction_guess = 0.5 + scale_factor_guess = 0.1 + scatter_peak_ratio_parameter_guess = 0.5 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + gauss_FWHM_eV_min = 1e-5 + gauss_FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess)-ConversionFunctions.Energy(bins_Hz[-1], B_field_guess) + survival_probability_min = 1e-5 + survival_probability_max = 1 + scatter_fraction_min = 1e-5 + scatter_fraction_max = 1 + scale_factor_min = 1e-5 + scale_factor_max = 5 + scatter_peak_ratio_parameter_min = 1e-5 + scatter_peak_ratio_parameter_max = 5 + N = len(self.gases) + gas_scatter_fraction_parameter_str = [] + for i in range(N-1): + gas_scatter_fraction_parameter_str += [self.gases[i]+' scatter fraction'] + p0_guess = [B_field_guess, amplitude_guess, gauss_FWHM_eV_guess, survival_probability_guess, scatter_peak_ratio_parameter_guess, scatter_peak_ratio_parameter_guess]+ (N-1)*[scatter_fraction_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (gauss_FWHM_eV_min, gauss_FWHM_eV_max), (survival_probability_min, survival_probability_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max)] + (N-1)*[(scatter_fraction_min, scatter_fraction_max)] + parameter_names = ['B field','amplitude','gaussian FWHM eV', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] + gas_scatter_fraction_parameter_str + # Actually do the fitting + m_binned = Minuit(lambda p: self.chi_2_gaussian_resolution_fit_scatter_peak_ratio(bins_Hz, data_hist_freq, eff_array, p), p0_guess, name = parameter_names) + m_binned.limits = p0_bounds + if len(self.fixed_parameter_names)>0: + for fixed_parameter_name, fixed_parameter_value in zip(self.fixed_parameter_names, self.fixed_parameter_values): + m_binned.fixed[fixed_parameter_name] = True + m_binned.values[fixed_parameter_name] = fixed_parameter_value + m_binned.errors[fixed_parameter_name] = 0 + m_binned.migrad() + m_binned.hesse() + params = m_binned.values[0:] + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + gauss_FWHM_eV_fit = params[2] + survival_probability_fit = params[3] + scatter_peak_ratio_b_fit = params[4] + scatter_peak_ratio_c_fit = params[5] + total_counts_fit = amplitude_fit + logger.info('\n'+str(m_binned.params)) + scatter_fraction_fit = params[6:5+N]+[1- sum(params[6:5+N])] + + perr = m_binned.errors[0:] + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + gauss_FWHM_eV_fit_err = perr[2] + survival_probability_fit_err = perr[3] + scatter_peak_ratio_b_fit_err = perr[4] + scatter_peak_ratio_c_fit_err = perr[5] + total_counts_fit_err = amplitude_fit_err + scatter_fraction_fit_err = perr[6:5+N]+[np.sqrt(sum(np.array(perr[6:5+N])**2))] + + fit_Hz = self.spectrum_func_gaussian_resolution_fit_scatter_peak_ratio(bins_Hz, eff_array, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + + if print_params == True: + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'gaussian FWHM = {:.8e}'.format(gauss_FWHM_eV_fit) + ' +/- {:.8e} eV\n'.format(gauss_FWHM_eV_fit_err) + output_string += '-----------------\n' + output_string += 'survival probability = {:.8e}'.format(survival_probability_fit) + ' +/- {:.8e}\n'.format(survival_probability_fit_err) + output_string += '-----------------\n' + output_string += 'scatter_peak_ratio_b = {:.8e}'.format(scatter_peak_ratio_b_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_b_fit_err) + output_string += '-----------------\n' + output_string += 'scatter_peak_ratio_c = {:.8e}'.format(scatter_peak_ratio_c_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_c_fit_err) + output_string += '-----------------\n' + for i in range(len(self.gases)): + output_string += '{} scatter fraction \n= '.format(self.gases[i]) + "{:.8e}".format(scatter_fraction_fit[i])\ + +' +/- ' + "{:.8e}".format(scatter_fraction_fit_err[i])+'\n' + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'gauss_FWHM_eV_fit': gauss_FWHM_eV_fit, + 'gauss_FWHM_eV_fit_err': gauss_FWHM_eV_fit_err, + 'scatter_peak_ratio_b_fit': scatter_peak_ratio_b_fit, + 'scatter_peak_ratio_b_fit_err': scatter_peak_ratio_b_fit_err, + 'scatter_peak_ratio_c_fit': scatter_peak_ratio_c_fit, + 'scatter_peak_ratio_c_fit_err': scatter_peak_ratio_c_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2 + } + + return dictionary_of_fit_results \ No newline at end of file From 5ba328025e2931da19601bb3de7ff0ed642784d7 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Thu, 24 Jun 2021 10:14:30 -0400 Subject: [PATCH 091/199] add 'gaussian_resolution_fit_scatter_peak_ratio' to resolution function instruction --- test_analysis/Complex_line_shape_fitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index e449a09e..6f8d6355 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -43,7 +43,7 @@ def test_complex_lineshape(self): 'fixed_survival_probability': False, # When option fixed_survival_probability is True, assign the survival probability below 'survival_prob': 15/16., # assuming total cross section for elastic scattering is 1/10 of inelastic scattering - # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, 'simulated_resolution_scaled_fit_scatter_peak_ratio' + # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, 'simulated_resolution_scaled_fit_scatter_peak_ratio', 'gaussian_resolution_fit_scatter_peak_ratio' 'resolution_function': 'simulated_resolution_scaled_fit_scatter_peak_ratio', # specific choice of parameters in the gaussian lorentzian composite resolution function 'recon_eff_param_a': 0.005569990343215976, From 76c1d9d8d84bddd18b79f19228688672be9c16f4 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Thu, 24 Jun 2021 10:38:19 -0400 Subject: [PATCH 092/199] minor modifications in Complex_line_shape_fitter.py --- test_analysis/Complex_line_shape_fitter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 6f8d6355..522e6bbf 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -58,8 +58,8 @@ def test_complex_lineshape(self): 'fit_recon_eff': False, #parameters for simulated resolution scaled with scatter peak ratio fitted #choose the parameters you want to fix from ['B field','amplitude','width scale factor', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] plus the gas scatter fractions as ['H2 scatter fraction'], - 'fixed_parameter_names': ['survival probability', 'H2 scatter fraction'], - 'fixed_parameter_values': [0.8, 0.9], + 'fixed_parameter_names': ['survival probability', 'H2 scatter fraction', 'width scale factor'], + 'fixed_parameter_values': [1.0, 0.896, 1.0], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, From c7c89da33e903d1248cb222ced32e56050698410 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Fri, 25 Jun 2021 15:46:30 -0400 Subject: [PATCH 093/199] copy the MultiGasComplexLineShape.py from combining_branch_temp_for_unresolved_conflicts branch --- .../misc/MultiGasComplexLineShape.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index bf57c482..2659be3d 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -79,7 +79,7 @@ def InternalConfigure(self, params): self.sigma_array = reader.read_param(params, 'sigma_array', [5.01, 13.33, 15.40, 11.85]) if self.resolution_function == 'simulated_resolution_scaled': self.fit_recon_eff = reader.read_param(params, 'fit_recon_eff', False) - if self.resolution_function == 'simulated_resolution_scaled_fit_scatter_peak_ratio': + if self.resolution_function == 'simulated_resolution_scaled_fit_scatter_peak_ratio' or 'gaussian_resolution_fit_scatter_peak_ratio': self.fixed_parameter_names = reader.read_param(params, 'fixed_parameter_names', []) self.fixed_parameter_values = reader.read_param(params, 'fixed_parameter_values', []) #self.elevation_factor = reader.read_param(params, 'elevation_factor', 20) @@ -158,6 +158,8 @@ def InternalRun(self): self.results = self.fit_data_simulated_resolution_scaled_fit_recon_eff(freq_bins, data_hist_freq) elif self.resolution_function == 'simulated_resolution_scaled_fit_scatter_peak_ratio': self.results = self.fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(freq_bins, data_hist_freq) + elif self.resolution_function == 'gaussian_resolution_fit_scatter_peak_ratio': + self.results = self.fit_data_gaussian_resolution_fit_scatter_peak_ratio(freq_bins, data_hist_freq) return True @@ -340,7 +342,6 @@ def generate_scatter_convolution_file(self): t = time.time() scatter_spectra_single_gas = {} for gas_type in self.gases: - f_radiation_loss = self.radiation_loss_f() scatter_spectra_single_gas[gas_type] = {} first_scatter = self.single_scatter_f(gas_type) if self.use_radiation_loss == True: @@ -352,6 +353,7 @@ def generate_scatter_convolution_file(self): for i in scatter_num_array: current_scatter = self.another_scatter(current_scatter, gas_type) if self.use_radiation_loss == True: + f_radiation_loss = self.radiation_loss_f() current_scatter = self.normalize(signal.convolve(current_scatter, f_radiation_loss, mode = 'same')) scatter_spectra_single_gas[gas_type][str(i).zfill(2)] = current_scatter N = len(self.gases) @@ -411,10 +413,10 @@ def check_existence_of_scatter_file(self, regenerate = True): return # Given a function evaluated on the SELA, convolves it with a gaussian - def convolve_gaussian(self, func_to_convolve,gauss_FWHM_eV): + def convolve_gaussian(self, func_to_convolve, gauss_FWHM_eV): sigma = ComplexLineShapeUtilities.gaussian_FWHM_to_sigma(gauss_FWHM_eV) resolution_f = self.std_gaussian(sigma) - ans = signal.convolve(resolution_f,func_to_convolve,mode='same') + ans = signal.convolve(resolution_f, func_to_convolve,mode='same') ans_normed = self.normalize(ans) return ans_normed @@ -3115,7 +3117,6 @@ def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_b*M**scatter_peak_ratio_c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: - #print(combination) entry_str = '' for component, gas_type in zip(combination, self.gases): entry_str += gas_type @@ -3176,14 +3177,14 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, eff_array = quad_trap_count_rate_interp(bins_Hz) # Initial guesses for curve_fit B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) - amplitude_guess = np.sum(data_hist_freq)/2 + amplitude_guess = np.sum(data_hist_freq) FWHM_eV_guess = 5 survival_probability_guess = 0.5 scatter_fraction_guess = 0.5 sigma_guess = 5 gamma_guess = 3 gaussian_portion_guess = 0.5 - scale_factor_guess = 1. + scale_factor_guess = 0.5 scatter_peak_ratio_parameter_guess = 0.5 # Bounds for curve_fit B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) @@ -3482,4 +3483,5 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi 'reduced_chi2': reduced_chi2 } - return dictionary_of_fit_results \ No newline at end of file + return dictionary_of_fit_results + From 00a65f742bbe5a5d74ecfd9574751ae7dcdc959f Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Fri, 25 Jun 2021 15:55:59 -0400 Subject: [PATCH 094/199] missed adding f_radiation_loss = self.radiation_loss_f() around line 348 --- mermithid/processors/misc/MultiGasComplexLineShape.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 2659be3d..fa7b880e 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -345,6 +345,7 @@ def generate_scatter_convolution_file(self): scatter_spectra_single_gas[gas_type] = {} first_scatter = self.single_scatter_f(gas_type) if self.use_radiation_loss == True: + f_radiation_loss = self.radiation_loss_f() first_scatter = self.normalize(signal.convolve(first_scatter, f_radiation_loss, mode = 'same')) scatter_num_array = range(2, self.max_scatters+1) current_scatter = first_scatter From 273522e13a5bbe2728014ceefc8464cb4d76abb5 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Fri, 25 Jun 2021 16:00:27 -0400 Subject: [PATCH 095/199] upload Complex_line_shape_fitter.py --- test_analysis/Complex_line_shape_fitter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 522e6bbf..e2cbfdfd 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -22,7 +22,7 @@ def test_complex_lineshape(self): reader_config = { "action": "read", - "filename": "/host/march_2020_kr_calibration_channel_b_merged.root", + "filename": "/host/shallow_trap_high_stats_above_10600_channel_a_concat.root", "object_type": "TMultiTrackEventData", "object_name": "multiTrackEvents:Event", "use_katydid": False, @@ -44,7 +44,7 @@ def test_complex_lineshape(self): # When option fixed_survival_probability is True, assign the survival probability below 'survival_prob': 15/16., # assuming total cross section for elastic scattering is 1/10 of inelastic scattering # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, 'simulated_resolution_scaled_fit_scatter_peak_ratio', 'gaussian_resolution_fit_scatter_peak_ratio' - 'resolution_function': 'simulated_resolution_scaled_fit_scatter_peak_ratio', + 'resolution_function': 'gaussian_resolution_fit_scatter_peak_ratio', # specific choice of parameters in the gaussian lorentzian composite resolution function 'recon_eff_param_a': 0.005569990343215976, 'recon_eff_param_b': 0.351, @@ -58,8 +58,8 @@ def test_complex_lineshape(self): 'fit_recon_eff': False, #parameters for simulated resolution scaled with scatter peak ratio fitted #choose the parameters you want to fix from ['B field','amplitude','width scale factor', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] plus the gas scatter fractions as ['H2 scatter fraction'], - 'fixed_parameter_names': ['survival probability', 'H2 scatter fraction', 'width scale factor'], - 'fixed_parameter_values': [1.0, 0.896, 1.0], + 'fixed_parameter_names': ['survival probability', 'H2 scatter fraction'], + 'fixed_parameter_values': [1.0, 0.896], # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown 'num_points_in_std_array': 10000, @@ -105,7 +105,7 @@ def test_complex_lineshape(self): plot_title = 'fit ftc march with gases: {},\n scatter proportion: {},\n resolution function: {},\n sigma_array: {},\n A_array: {},\n'.format(complexLineShape_config['gases'], complexLineShape_config['gas_scatter_proportion'], complexLineShape_config['resolution_function'], complexLineShape_config['sigma_array'], complexLineShape_config['A_array']) plt.title(plot_title) plt.tight_layout() - plt.savefig('/host/plots/fit_FTC_march_with_simulated_ins_scaled_resolution.png'.format(len(complexLineShape_config['gases']))) + plt.savefig('/host/plots/fit_shallow_trap_above_10600_with_gaussian_resolution.png'.format(len(complexLineShape_config['gases']))) if __name__ == '__main__': From 347fb4984f91e0d2997415f0542cdb5515b13ae4 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Mon, 28 Jun 2021 13:15:04 -0400 Subject: [PATCH 096/199] Made gaussian res an option in FakeDataGenerator --- mermithid/misc/FakeTritiumDataFunctions.py | 16 +++++++++++++--- .../TritiumSpectrum/FakeDataGenerator.py | 4 ++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index e1de6b3c..53396735 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -275,7 +275,7 @@ def convolved_bkgd_rate(K, Kmin, Kmax, lineshape, ls_params, min_energy, max_ene #Convolution of signal and lineshape using scipy.signal.convolve def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, min_energy, max_energy, - complexLineShape, final_state_array): + complexLineShape, final_state_array, resolution_function): """K is an array-like object """ logger.info('Using scipy convolve') @@ -293,7 +293,12 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(1, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') + if resolution_function == 'simulated_resolution' or 'simulated': + lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(1, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') + elif resolution_function == 'gaussian_resolution' or 'gaussian': + lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') + else: + logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) lineshape_rates = np.flipud(lineshape_rates) beta_rates = spectral_rate(K, Q, mnu, final_state_array) @@ -324,7 +329,12 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(1, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') + if resolution_function == 'simulated_resolution' or 'simulated': + lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(1, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') + elif resolution_function == 'gaussian_resolution' or 'gaussian': + lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') + else: + logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) lineshape_rates = np.flipud(lineshape_rates) bkgd_rates = np.full(len(K), bkgd_rate()) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 1858f931..51445154 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -375,7 +375,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 if array_method == True: ratesS = convolved_spectral_rate_arrays(self.Koptions, Q_mean, mass, Kmin, lineshape, params, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, self.scatter_proportion, min_energy, max_energy, - self.complexLineShape, self.final_state_array) + self.complexLineShape, self.final_state_array, self.resolution_function) else: ratesS = [convolved_spectral_rate(K, Q_mean, mass, Kmin, lineshape, params, min_energy, max_energy) for K in @@ -391,7 +391,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 if array_method == True: ratesB = convolved_bkgd_rate_arrays(self.Koptions, Kmin, Kmax, lineshape, params, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, self.scatter_proportion, min_energy, max_energy, - self.complexLineShape) + self.complexLineShape, self.resolution_function) else: ratesB = [convolved_bkgd_rate(K, Kmin, Kmax, lineshape, params, min_energy, max_energy) for K in self.Koptions] From 29eb20b95cb6d0a87cca7e4d213e91eb0a7d0e99 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Wed, 30 Jun 2021 14:55:16 -0400 Subject: [PATCH 097/199] Fixed background function for FakeDataGenerator --- Dockerfile | 6 ++---- mermithid/misc/FakeTritiumDataFunctions.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index dbde3af5..1e489c2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,10 +18,6 @@ RUN mkdir -p $MERMITHID_BUILD_PREFIX &&\ echo 'export PYTHONPATH=$MERMITHID_BUILD_PREFIX/$(python -m site --user-site | sed "s%$(python -m site --user-base)%%"):$PYTHONPATH' >> setup.sh &&\ /bin/true -RUN source $COMMON_BUILD_PREFIX/setup.sh &&\ - pip install iminuit &&\ - /bin/true - ######################## FROM mermithid_common as mermithid_done @@ -41,6 +37,7 @@ COPY tests $MERMITHID_BUILD_PREFIX/tests # repeat the cmake command to get the change of install prefix to set correctly (a package_builder known issue) RUN source $MERMITHID_BUILD_PREFIX/setup.sh &&\ + pip3 install --upgrade pip &&\ cd /tmp_source &&\ mkdir -p build &&\ cd build &&\ @@ -54,6 +51,7 @@ RUN source $MERMITHID_BUILD_PREFIX/setup.sh &&\ cd /tmp_source &&\ # ls -altrh morpho &&\ pip3 install . ./morpho --prefix $MERMITHID_BUILD_PREFIX &&\ + pip3 install iminuit &&\ /bin/true ######################## diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 53396735..60eed5dc 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -313,7 +313,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, #Convolution of background and lineshape using scipy.signal.convolve -def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, min_energy, max_energy, complexLineShape): +def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, min_energy, max_energy, complexLineShape, resolution_function): """K is an array-like object """ energy_half_range = max(max_energy, abs(min_energy)) From 703e8bc5e1680b49e1d66515455bd65cdb787815 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Thu, 1 Jul 2021 17:13:31 -0400 Subject: [PATCH 098/199] Temporarily added diagnostic plotting functions --- mermithid/misc/FakeTritiumDataFunctions.py | 17 +++++++++++++++++ .../TritiumSpectrum/FakeDataGenerator.py | 2 ++ 2 files changed, 19 insertions(+) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 60eed5dc..af0201bb 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -20,6 +20,8 @@ from mermithid.misc.Constants import * from mermithid.misc.ConversionFunctions import * +import matplotlib.pyplot as plt + """ Constants and functions used by processors/TritiumSpectrum/FakeDataGenerator.py """ @@ -301,11 +303,26 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) lineshape_rates = np.flipud(lineshape_rates) + fig = plt.figure() + plt.plot(complexLineShape.std_eV_array(), lineshape_rates) + plt.xlabel('Energy shift (eV)', fontsize=15) + plt.ylabel('Response function (a.u.)', fontsize=15) + plt.xlim([-750., 500.]) + plt.savefig('complex_lineshape_rates_{}.pdf'.format(ls_params[1])) + plt.show() + beta_rates = spectral_rate(K, Q, mnu, final_state_array) #Convolving convolved = convolve(beta_rates, lineshape_rates, mode='same') + #fig = plt.figure() + #plt.plot(K, convolved) + #plt.xlabel('Energy (eV)') + #plt.ylabel('Signal rate') + #plt.savefig('spectrum_signal.pdf') + #plt.show() + below_Kmin = np.where(K < Kmin) np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) return convolved diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 51445154..5c332890 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -341,6 +341,8 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 else: Kmin, Kmax = ROIbound[0], ROIbound[1] B = B_1kev*(Kmax-Kmin)/1000. + print("MIN KE:", Kmin) + print("MAX F:", maxf) nstdevs = 7 #Number of standard deviations (of size broadening) below Kmin and above Q-m to generate data, for the gaussian case FWHM_convert = 2*math.sqrt(2*math.log(2)) From 36d1d81a96f5ababef79ec35fca5a57d139cebc2 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Tue, 20 Jul 2021 16:06:41 -0400 Subject: [PATCH 099/199] Fixed bug in Gaussian response fn option --- mermithid/misc/FakeTritiumDataFunctions.py | 5 +++-- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index af0201bb..82982bef 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -292,6 +292,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, if lineshape=='gaussian': logger.info('broadening: {}'.format(ls_params[0])) lineshape_rates = gaussian(K_lineshape, [ls_params[0], 0]) + print(ls_params[0]) elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': @@ -304,11 +305,11 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape_rates = np.flipud(lineshape_rates) fig = plt.figure() - plt.plot(complexLineShape.std_eV_array(), lineshape_rates) + plt.plot(K_lineshape, lineshape_rates) #complexLineShape.std_eV_array() plt.xlabel('Energy shift (eV)', fontsize=15) plt.ylabel('Response function (a.u.)', fontsize=15) plt.xlim([-750., 500.]) - plt.savefig('complex_lineshape_rates_{}.pdf'.format(ls_params[1])) + plt.savefig('lineshape_rates.pdf')#.format(ls_params[1])) plt.show() beta_rates = spectral_rate(K, Q, mnu, final_state_array) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 5c332890..371ea661 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -213,7 +213,7 @@ def InternalConfigure(self, params): 'sigma_array': self.sigma_array, # This is an important parameter which determines how finely resolved the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to be increased for larger datasets. - 'num_points_in_std_array': 35838, + 'num_points_in_std_array': 150000,#35838, 'base_shape': 'dirac', 'path_to_osc_strengths_files': self.path_to_osc_strengths_files, 'path_to_scatter_spectra_file':self.path_to_scatter_spectra_file, @@ -235,7 +235,7 @@ def InternalConfigure(self, params): else: self.lineshape = 'gaussian' - self.SimpParams = [self.broadening] + self.SimpParams = [self.scattering_sigma] logger.info('Lineshape is Gaussian') # check final states file existence From 83d61068b69087ec722e260c69b96868e0d41f29 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 28 Jul 2021 06:25:59 -0400 Subject: [PATCH 100/199] add configuration use_quad_eff_interp --- .../misc/MultiGasComplexLineShape.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index fa7b880e..59b6b449 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -95,7 +95,9 @@ def InternalConfigure(self, params): self.path_to_ins_resolution_data_txt = reader.read_param(params, 'path_to_ins_resolution_data_txt', '/host/res_cf15.5_all.txt') self.use_combined_four_trap_inst_reso = reader.read_param(params, 'use_combined_four_trap_inst_reso', False) self.path_to_four_trap_ins_resolution_data_txt = reader.read_param(params, 'path_to_four_trap_ins_resolution_data_txt', ['/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap1.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap2.txt', '/host/T2-1.56e-4/analysis_input/complex-lineshape-inputs/res_cf15.5_trap3.txt', '/host/analysis_input/complex-lineshape-inputs/T2-1.56e-4/res_cf15.5_trap4.txt']) - self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') + self.use_quad_trap_eff_interp = reader.read_param(params, 'use_quad_trap_eff_interp', True) + if self.use_quad_trap_eff_interp == True: + self.path_to_quad_trap_eff_interp = reader.read_param(params, 'path_to_quad_trap_eff_interp', '/host/quad_interps.npy') self.recon_eff_params = reader.read_param(params, 'recon_eff_params', [0.005569990343215976, 0.351, 0.546]) self.recon_eff_param_a = self.recon_eff_params[0] self.recon_eff_param_b = self.recon_eff_params[1] @@ -3173,9 +3175,12 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) - quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] - eff_array = quad_trap_count_rate_interp(bins_Hz) + if self.use_quad_trap_eff_interp == True: + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + else: + eff_array = np.ones(len(bins_Hz)) # Initial guesses for curve_fit B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) amplitude_guess = np.sum(data_hist_freq) @@ -3369,10 +3374,13 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi t = time.time() self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN - bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) - quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] - eff_array = quad_trap_count_rate_interp(bins_Hz) + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + if self.use_quad_trap_eff_interp == True: + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + else: + eff_array = np.ones(len(bins_Hz)) # Initial guesses for curve_fit B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) amplitude_guess = np.sum(data_hist_freq)/2 From 5f32964b1e3295f205048cea115d64d5c3c7083e Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Wed, 28 Jul 2021 14:09:57 -0400 Subject: [PATCH 101/199] Changed survival_prob default to 1 --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 371ea661..6719a581 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -125,7 +125,7 @@ def InternalConfigure(self, params): self.scatter_peak_ratio_b = reader.read_param(params, 'scatter_peak_ratio_b', 0.686312493) self.scatter_peak_ratio_c = reader.read_param(params, 'scatter_peak_ratio_c', 0.52481056) self.scatter_proportion = reader.read_param(params, 'scatter_proportion', []) - self.survival_prob = reader.read_param(params, 'survival_prob', 0.7) + self.survival_prob = reader.read_param(params, 'survival_prob', 1.) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) self.resolution_function = reader.read_param(params, 'resolution_function', '') self.ratio_gamma_to_sigma = reader.read_param(params, 'ratio_gamma_to_sigma', 0.8) From ed9628474bd6e3ca8aafc5e51f1e6063b7bee7f6 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Wed, 28 Jul 2021 16:55:25 -0400 Subject: [PATCH 102/199] Updated len(std_eV_array) for Phase II --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 6719a581..b344b46b 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -213,7 +213,7 @@ def InternalConfigure(self, params): 'sigma_array': self.sigma_array, # This is an important parameter which determines how finely resolved the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to be increased for larger datasets. - 'num_points_in_std_array': 150000,#35838, + 'num_points_in_std_array':35846, 'base_shape': 'dirac', 'path_to_osc_strengths_files': self.path_to_osc_strengths_files, 'path_to_scatter_spectra_file':self.path_to_scatter_spectra_file, From 92750ea674a107c6eea741b52dcc255d05051247 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Sun, 1 Aug 2021 11:46:52 -0400 Subject: [PATCH 103/199] Added scale factor to fake data generator --- mermithid/misc/FakeTritiumDataFunctions.py | 4 ++-- .../TritiumSpectrum/FakeDataGenerator.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 82982bef..7644f45b 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -279,7 +279,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, min_energy, max_energy, complexLineShape, final_state_array, resolution_function): """K is an array-like object - """ + """ logger.info('Using scipy convolve') energy_half_range = max(max_energy, abs(min_energy)) @@ -297,7 +297,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': if resolution_function == 'simulated_resolution' or 'simulated': - lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(1, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') + lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') elif resolution_function == 'gaussian_resolution' or 'gaussian': lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') else: diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index b344b46b..c1a28a1f 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -69,6 +69,7 @@ def InternalConfigure(self, params): - sample_ins_resolution_errors: if True, and if using a simulated instrumental resolution, the count numbers in that distribution will be sampled from uncertainties - scattering_sigma [eV]: lineshape parameter - 0-th peak gaussian broadening standard deviation - min_energy [eV]: minimum of lineshape energy window for convolution with beta spectrum. Same magnitude is used for the max energy of the window. + - scale_factor: width scaling for a simulated instrumental resolution - efficiency_path: path to efficiency vs. frequency (and uncertainties) - simplified_scattering_path: path to simplified lineshape parameters – path_to_osc_strengths_files: path to oscillator strength files containing energy loss distributions for the gases in self.gases @@ -137,6 +138,7 @@ def InternalConfigure(self, params): self.sample_ins_resolution_errors = reader.read_param(params, 'sample_ins_res_errors', True) self.scattering_sigma = reader.read_param(params, 'scattering_sigma', 18.6) self.min_energy = reader.read_param(params,'min_lineshape_energy', -1000) + self.scale_factor = reader.read_param(params, 'scale_factor', 1.0) #paths self.efficiency_path = reader.read_param(params, 'efficiency_path', '') @@ -172,7 +174,7 @@ def InternalConfigure(self, params): if self.use_lineshape: self.lineshape = self.detailed_or_simplified_lineshape if self.lineshape == 'simplified': - self.SimpParams = self.load_simp_params(self.scattering_sigma, + self.ls_params = self.load_simp_params(self.scattering_sigma, self.survival_prob, self.NScatters) elif self.lineshape=='detailed': @@ -187,7 +189,8 @@ def InternalConfigure(self, params): # lineshape params - self.SimpParams = [self.scattering_sigma*2*math.sqrt(2*math.log(2)), self.survival_prob] + self.ls_params = [self.scale_factor, self.survival_prob] + # Setup and configure lineshape processor complexLineShape_config = { 'gases': self.gases, @@ -235,7 +238,7 @@ def InternalConfigure(self, params): else: self.lineshape = 'gaussian' - self.SimpParams = [self.scattering_sigma] + self.ls_params = [self.scattering_sigma] logger.info('Lineshape is Gaussian') # check final states file existence @@ -268,7 +271,7 @@ def InternalRun(self): self.S, self.B_1kev, nsteps=self.n_steps, lineshape=self.lineshape, - params=self.SimpParams, + params=self.ls_params, efficiency_dict = self.efficiency_dict, err_from_B=self.err_from_B, B_field=self.B_field) @@ -341,9 +344,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 else: Kmin, Kmax = ROIbound[0], ROIbound[1] B = B_1kev*(Kmax-Kmin)/1000. - print("MIN KE:", Kmin) - print("MAX F:", maxf) - + nstdevs = 7 #Number of standard deviations (of size broadening) below Kmin and above Q-m to generate data, for the gaussian case FWHM_convert = 2*math.sqrt(2*math.log(2)) max_energy = -self.min_energy From 5cb8e159888e8fcdfc991949bdf63f2b7584eb27 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Sun, 1 Aug 2021 23:44:45 -0400 Subject: [PATCH 104/199] Allow energy res to vary with K in data gen --- mermithid/misc/FakeTritiumDataFunctions.py | 63 +++++++++++++------ .../TritiumSpectrum/FakeDataGenerator.py | 4 +- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 7644f45b..d076dccb 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -274,15 +274,25 @@ def convolved_bkgd_rate(K, Kmin, Kmax, lineshape, ls_params, min_energy, max_ene return 0. + #Convolution of signal and lineshape using scipy.signal.convolve def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, min_energy, max_energy, - complexLineShape, final_state_array, resolution_function): + complexLineShape, final_state_array, resolution_function, ins_res_width_bounds, ins_res_width_factors): """K is an array-like object - """ + """ logger.info('Using scipy convolve') energy_half_range = max(max_energy, abs(min_energy)) + if ins_res_width_bounds != None: + Kbounds = [np.min(K)] + ins_res_width_bounds + [np.max(K)] + else: + Kbounds = [np.min(K), np.max(K)] + + K_segments = [] + for i in range(len(Kbounds)-1): + K_segments.append(K[np.logical_and(Kbounds[i]<=K, K<=Kbounds[i+1])]) + dE = K[1] - K[0] n_dE_pos = round(energy_half_range/dE) #Number of steps for the lineshape for energies > 0 n_dE_neg = round(energy_half_range/dE) #Same, for energies < 0 @@ -292,37 +302,54 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, if lineshape=='gaussian': logger.info('broadening: {}'.format(ls_params[0])) lineshape_rates = gaussian(K_lineshape, [ls_params[0], 0]) - print(ls_params[0]) elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': if resolution_function == 'simulated_resolution' or 'simulated': - lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') + lineshape_rates = [] + scale_factors = [ls_params[0]*f for f in ins_res_width_factors] + for scale in scale_factors: + lineshape_rates.append(complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac')) elif resolution_function == 'gaussian_resolution' or 'gaussian': lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') else: logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) lineshape_rates = np.flipud(lineshape_rates) - + """ fig = plt.figure() - plt.plot(K_lineshape, lineshape_rates) #complexLineShape.std_eV_array() + plt.plot(K_lineshape, lineshape_rates[0]) #complexLineShape.std_eV_array() plt.xlabel('Energy shift (eV)', fontsize=15) plt.ylabel('Response function (a.u.)', fontsize=15) plt.xlim([-750., 500.]) - plt.savefig('lineshape_rates.pdf')#.format(ls_params[1])) + plt.savefig('lineshape_rates_{}.pdf'.format(scale_factors[0])) plt.show() - - beta_rates = spectral_rate(K, Q, mnu, final_state_array) + + fig = plt.figure() + plt.plot(K_lineshape, lineshape_rates[1]) + plt.xlabel('Energy shift (eV)', fontsize=15) + plt.ylabel('Response function (a.u.)', fontsize=15) + plt.xlim([-750., 500.]) + plt.savefig('lineshape_rates_{}.pdf'.format(scale_factors[1])) + plt.show() + """ #Convolving - convolved = convolve(beta_rates, lineshape_rates, mode='same') - - #fig = plt.figure() - #plt.plot(K, convolved) - #plt.xlabel('Energy (eV)') - #plt.ylabel('Signal rate') - #plt.savefig('spectrum_signal.pdf') - #plt.show() + len_beta_rates = 0 + len_convolved_list = 0 + if lineshape=='detailed_scattering' or lineshape=='detailed': + if resolution_function == 'simulated_resolution' or 'simulated': + convolved_list = [] + for j in range(len(lineshape_rates)): + beta_rates = spectral_rate(K_segments[j], Q, mnu, final_state_array) + convolved_list.append(convolve(beta_rates, lineshape_rates[j], mode='same')) + len_beta_rates += len(beta_rates) + len_convolved_list += len(convolved_list[j]) + convolved = np.concatenate(convolved_list, axis=None) + + else: + beta_rates = spectral_rate(K, Q, mnu, final_state_array) + convolved = convolve(beta_rates, lineshape_rates, mode='same') + below_Kmin = np.where(K < Kmin) np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) @@ -348,7 +375,7 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': if resolution_function == 'simulated_resolution' or 'simulated': - lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(1, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') + lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') elif resolution_function == 'gaussian_resolution' or 'gaussian': lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') else: diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index c1a28a1f..a002c3b8 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -139,6 +139,8 @@ def InternalConfigure(self, params): self.scattering_sigma = reader.read_param(params, 'scattering_sigma', 18.6) self.min_energy = reader.read_param(params,'min_lineshape_energy', -1000) self.scale_factor = reader.read_param(params, 'scale_factor', 1.0) + self.ins_res_width_bounds = reader.read_param(params, 'ins_res_width_bounds', [19430., 19225., 18712., 18507., 17384., 17180.]) #Default values here need to be corrected + self.ins_res_width_factors = reader.read_param(params, 'ins_res_width_bounds', [1, 1.125, 1, 0.825, 1, 0.925]) #paths self.efficiency_path = reader.read_param(params, 'efficiency_path', '') @@ -378,7 +380,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 if array_method == True: ratesS = convolved_spectral_rate_arrays(self.Koptions, Q_mean, mass, Kmin, lineshape, params, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, self.scatter_proportion, min_energy, max_energy, - self.complexLineShape, self.final_state_array, self.resolution_function) + self.complexLineShape, self.final_state_array, self.resolution_function, self.ins_res_width_bounds, self.ins_res_width_factors) else: ratesS = [convolved_spectral_rate(K, Q_mean, mass, Kmin, lineshape, params, min_energy, max_energy) for K in From 58400bd6b5b00352c60a34215c944d2443a6b53a Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Mon, 2 Aug 2021 10:18:14 -0400 Subject: [PATCH 105/199] Cleaning - ins res variation with K in data gen --- mermithid/misc/FakeTritiumDataFunctions.py | 8 ++++---- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index d076dccb..0c08febd 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -309,13 +309,12 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape_rates = [] scale_factors = [ls_params[0]*f for f in ins_res_width_factors] for scale in scale_factors: - lineshape_rates.append(complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac')) + lineshape_rates.append(np.flipud(complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac'))) elif resolution_function == 'gaussian_resolution' or 'gaussian': lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') else: logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) - lineshape_rates = np.flipud(lineshape_rates) - """ + fig = plt.figure() plt.plot(K_lineshape, lineshape_rates[0]) #complexLineShape.std_eV_array() plt.xlabel('Energy shift (eV)', fontsize=15) @@ -331,7 +330,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, plt.xlim([-750., 500.]) plt.savefig('lineshape_rates_{}.pdf'.format(scale_factors[1])) plt.show() - """ + #Convolving len_beta_rates = 0 @@ -347,6 +346,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, convolved = np.concatenate(convolved_list, axis=None) else: + lineshape_rates = np.flipud(lineshape_rates) beta_rates = spectral_rate(K, Q, mnu, final_state_array) convolved = convolve(beta_rates, lineshape_rates, mode='same') diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index a002c3b8..94121c15 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -139,8 +139,8 @@ def InternalConfigure(self, params): self.scattering_sigma = reader.read_param(params, 'scattering_sigma', 18.6) self.min_energy = reader.read_param(params,'min_lineshape_energy', -1000) self.scale_factor = reader.read_param(params, 'scale_factor', 1.0) - self.ins_res_width_bounds = reader.read_param(params, 'ins_res_width_bounds', [19430., 19225., 18712., 18507., 17384., 17180.]) #Default values here need to be corrected - self.ins_res_width_factors = reader.read_param(params, 'ins_res_width_bounds', [1, 1.125, 1, 0.825, 1, 0.925]) + self.ins_res_width_bounds = reader.read_param(params, 'ins_res_width_bounds', [17384.]) #Default values here need to be corrected #[17180., 17384., 18507., 18712., 19225., 19430.] + self.ins_res_width_factors = reader.read_param(params, 'ins_res_width_bounds', [0.9, 1.1]) #[1, 1.125, 1, 0.825, 1, 0.925] #paths self.efficiency_path = reader.read_param(params, 'efficiency_path', '') From 63efaf0c72a80bd870edfd881da21531ca8f5219 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Mon, 2 Aug 2021 10:50:36 -0400 Subject: [PATCH 106/199] Changed ins res width defaults; fixed error --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 94121c15..83b13e5a 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -139,8 +139,8 @@ def InternalConfigure(self, params): self.scattering_sigma = reader.read_param(params, 'scattering_sigma', 18.6) self.min_energy = reader.read_param(params,'min_lineshape_energy', -1000) self.scale_factor = reader.read_param(params, 'scale_factor', 1.0) - self.ins_res_width_bounds = reader.read_param(params, 'ins_res_width_bounds', [17384.]) #Default values here need to be corrected #[17180., 17384., 18507., 18712., 19225., 19430.] - self.ins_res_width_factors = reader.read_param(params, 'ins_res_width_bounds', [0.9, 1.1]) #[1, 1.125, 1, 0.825, 1, 0.925] + self.ins_res_width_bounds = reader.read_param(params, 'ins_res_width_bounds', None) #Default values here need to be corrected + self.ins_res_width_factors = reader.read_param(params, 'ins_res_width_factors', [1]) #paths self.efficiency_path = reader.read_param(params, 'efficiency_path', '') From c56ea199221a02da56559c9233a743669b034f72 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Mon, 2 Aug 2021 12:05:29 -0400 Subject: [PATCH 107/199] Removed temporary plotting and printing --- mermithid/misc/FakeTritiumDataFunctions.py | 23 +--------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 0c08febd..255f8712 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -314,37 +314,16 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') else: logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) - - fig = plt.figure() - plt.plot(K_lineshape, lineshape_rates[0]) #complexLineShape.std_eV_array() - plt.xlabel('Energy shift (eV)', fontsize=15) - plt.ylabel('Response function (a.u.)', fontsize=15) - plt.xlim([-750., 500.]) - plt.savefig('lineshape_rates_{}.pdf'.format(scale_factors[0])) - plt.show() - - fig = plt.figure() - plt.plot(K_lineshape, lineshape_rates[1]) - plt.xlabel('Energy shift (eV)', fontsize=15) - plt.ylabel('Response function (a.u.)', fontsize=15) - plt.xlim([-750., 500.]) - plt.savefig('lineshape_rates_{}.pdf'.format(scale_factors[1])) - plt.show() - #Convolving - len_beta_rates = 0 - len_convolved_list = 0 if lineshape=='detailed_scattering' or lineshape=='detailed': if resolution_function == 'simulated_resolution' or 'simulated': convolved_list = [] for j in range(len(lineshape_rates)): beta_rates = spectral_rate(K_segments[j], Q, mnu, final_state_array) convolved_list.append(convolve(beta_rates, lineshape_rates[j], mode='same')) - len_beta_rates += len(beta_rates) - len_convolved_list += len(convolved_list[j]) convolved = np.concatenate(convolved_list, axis=None) - + else: lineshape_rates = np.flipud(lineshape_rates) beta_rates = spectral_rate(K, Q, mnu, final_state_array) From dacc7849c7e21fa953c1d99e2d6759107ed4a698 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Tue, 3 Aug 2021 22:57:12 -0700 Subject: [PATCH 108/199] added tritium frequentists analysis --- .../misc/DetectionEfficiencyUtilities.py | 280 ++++ mermithid/misc/__init__.py | 1 + .../Fitters/MCUncertaintyPropagation.py | 150 ++ mermithid/processors/Fitters/__init__.py | 1 + .../TritiumSpectrum/BinnedTritiumMLFitter.py | 1289 +++++++++++++++++ .../processors/TritiumSpectrum/__init__.py | 1 + 6 files changed, 1722 insertions(+) create mode 100755 mermithid/misc/DetectionEfficiencyUtilities.py create mode 100644 mermithid/processors/Fitters/MCUncertaintyPropagation.py create mode 100755 mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py diff --git a/mermithid/misc/DetectionEfficiencyUtilities.py b/mermithid/misc/DetectionEfficiencyUtilities.py new file mode 100755 index 00000000..6a6c7ab9 --- /dev/null +++ b/mermithid/misc/DetectionEfficiencyUtilities.py @@ -0,0 +1,280 @@ + + +import numpy as np +import matplotlib.pyplot as plt +import os +from scipy.interpolate import interp1d +from scipy.stats import binom +from scipy import constants +from statsmodels.stats.proportion import proportion_confint +import helper_functions.alis_power as ap +import helper_functions.plot_methods as pm + + +def binomial_interval(n, k, alpha=1 / 3.1514872): + + if isinstance(n, list) or type(n) is np.ndarray: + + mean = np.zeros(len(n)) + interval = np.zeros((2, len(n))) + interval_mean = np.zeros(len(n)) + for i in range(len(n)): + interval[0][i], interval[1][i] = proportion_confint(k[i], n[i], method='wilson', alpha=alpha) + rv = binom(n[i], k[i]*1./n[i]) + mean[i] = rv.mean()*1./n[i] + interval_mean[i] = np.mean(interval.T[i]) + interval_error = [interval_mean - interval[0], interval[1]-interval_mean] + + else: + l_int, u_int = proportion_confint(k, n, method='wilson', alpha=alpha) + rv = binom(n, k*1./n) + mean = rv.mean()*1./n + interval_mean = np.mean([l_int, u_int]) + interval_error = [interval_mean - l_int, u_int-interval_mean] + return mean, interval_mean, interval_error + + +def freq2en(f, mixfreq=24.5e9): + """ + convert IF frequency in MHz to energy in keV + """ + B=0.9578170819250281 + f = f*1e6 + emass = constants.electron_mass/constants.e*constants.c**2 + gamma = (constants.e*B)/(2.0*np.pi*constants.electron_mass) * 1/(f+mixfreq) + + return (gamma -1)*emass*1e-3 + + +def en2freq(E, Theta=None, mixfreq=24.5e9): + """ + convert energy in keV to IF frequency in MHz + """ + B=0.9578170819250281 + E = E*1e3 + if Theta==None: + Theta=np.pi/2 + e = constants.e + c = constants.c + + emass = constants.electron_mass/constants.e*constants.c**2 + gamma = E/(emass)+1 + + return ((constants.e*B)/(2.0*np.pi*constants.electron_mass) * 1/gamma)*1e-6-mixfreq*1e-6 + + +def power_efficiency(f, plot=False, savepath='.'): + """ + Toy model efficiency using pheno paper power vs. energy and arbitrary threshold + """ + #print('Toy model efficiency') + threshold = 8 + N = int(1e6) + efficiency = np.zeros(len(f)) + efficiency_error = np.zeros((len(f),2)) + power_factor = ap.Power(f)/ap.Power([1407e6+24.5e9]) + + #if plot: + # N = int(1e6) + + data0 = np.random.exponential(2, N) + + if plot: + plt.figure(figsize=(7,5)) + plt.hist(data0, histtype='step', bins=100, color=pm.sns_color[0]) + plt.axvline(threshold, label='Detection threshold', color=pm.sns_color[4]) + plt.ylabel('N') + plt.legend() + plt.xlabel('SNR') + plt.yscale('log') + plt.tight_layout() + plt.savefig(os.path.join(savepath, 'hypothetical_detection.png'), dpi=200, transparent=True) + plt.savefig(os.path.join(savepath, 'hypothetical_detection.pdf'), dpi=200, transparent=True) + + for i in range(len(f)): + + data = data0*power_factor[i] + efficiency[i], _, efficiency_error[i] = binomial_interval(len(data), len(data[data>threshold])) + + + if plot: + plt.figure(figsize=(7,5)) + plt.plot(freq2en(f*1e-6-24.5e3), efficiency, color=pm.sns_color[0]) + #plt.fill_between(freq2en(f*1e-6-24.5e3), efficiency-efficiency_error.T[0], efficiency+efficiency_error.T[1], color=pm.sns_color[0], alpha=0.5) + plt.ylabel('Efficiency') + plt.xlabel('Energy [keV]') + plt.tight_layout() + + #print((efficiency[-1]-efficiency[0])/(freq2en(f*1e-6-24.5e3)[-1]-freq2en(f*1e-6-24.5e3)[0])/efficiency[0]) + plt.savefig(os.path.join(savepath, 'hypothetical_efficiency.png'), dpi=200, transparent=True) + plt.savefig(os.path.join(savepath, 'hypothetical_efficiency.pdf'), dpi=200, transparent=True) + + return efficiency/np.mean(efficiency), efficiency_error.T/np.mean(efficiency) + +def interpolated_efficiency(f, snr_efficiency_dict, alpha=1): + """ + turn efficiencies at fixed frequencies into callable efficiency for arbitrary frequency + alpha scales uncertainty + """ + + # only use good index (where fits in efficiency analysis didn't fail) + index_0 = snr_efficiency_dict['good_fit_index'] + + x = np.array(snr_efficiency_dict['frequency'])[index_0] + y = np.array(snr_efficiency_dict['tritium_rates'])[index_0] + y = [y for _,y in sorted(zip(x,y))] + y_err_0 = np.array(snr_efficiency_dict['tritium_rates_error'][0])[index_0]*alpha + y_err_1 = np.array(snr_efficiency_dict['tritium_rates_error'][1])[index_0]*alpha + y_err_0 = [y_err for _,y_err in sorted(zip(x,y_err_0))] + y_err_1 = [y_err for _,y_err in sorted(zip(x,y_err_1))] + x = sorted(x) + + yp = np.interp(f, x, y, left=0, right=0) + yp_error_0 = np.interp(f, x, y_err_0, left=1, right=1) + yp_error_1 = np.interp(f, x, y_err_1, left=1, right=1) + + + return np.array(yp), np.array([yp_error_0, yp_error_1]) + +def pseudo_interpolated_efficiency(f, snr_efficiency_dict, alpha=1): + """ + gaussian random efficiency in uncertainty intervall + """ + if isinstance(f, float): + y, y_error = interpolated_efficiency(f, snr_efficiency_dict,alpha) + pseudo_y = np.random.randn()*y_error + y + else: + y, y_error = interpolated_efficiency(f, snr_efficiency_dict, alpha) + + pseudo_y = np.random.randn(len(f)) + pseudo_y[pseudo_y<0]*=np.array(y_error)[0][pseudo_y<0] + pseudo_y[pseudo_y>0]*=np.array(y_error)[1][pseudo_y>0] + + + return pseudo_y+y, y_error + + +def integrated_efficiency(f, snr_efficiency_dict, df=None , mix_freq=24.5e9, centers = True): + """ + pseudo integrates interpolated efficiency over bin width. + integral is approximated by just averaging a few points over the bin, because its much faster + + f: frequency bins in absolute frequencies (>25GHz), units is Hz + df: width around bin centers that should be integrated over. Only used if f is float + mix_freq: required because efficiency analysis was done in IF frequencies + center: if true assume f is bin centers, if False assume f is bin edges + """ + + + # number of points summed in "integration" + N = 10 + + # inteprolate efficiency and efficiency uncertainty + + frequency = np.array(snr_efficiency_dict['frequency'])[snr_efficiency_dict['good_fit_index']] + efficiency, efficiency_error = interpolated_efficiency(frequency, snr_efficiency_dict) + + interp_eff = interp1d(frequency-24.5e9, efficiency, fill_value=0, bounds_error=False) + interp_eff_error_up = interp1d(frequency-24.5e9, efficiency_error[1], fill_value=1, bounds_error=False) + interp_eff_error_down = interp1d(frequency-24.5e9, efficiency_error[0], fill_value=1, bounds_error=False) + + + # bin width + if isinstance(f, float): + if df==None: + raise Exception('No integration width given') + + + # if df is smaller than FSS bin width, do not integrate but just return interpolation + + if df > 2e6: + x = np.linspace(f-mix_freq-df/2, f-mix_freq+df/2, N) + eff = np.sum(interp_eff(x))/N + eff_err_down = np.sum(interp_eff_error_down(x))/N + eff_err_up = np.sum(interp_eff_error_up(x))/N + return eff, [eff_err_down, eff_err_down] + + else: + eff = interp_eff(f-mix_freq) + eff_error_down = interp_eff_error_down(f-mix_freq) + eff_error_up = interp_eff_error_up(f-mix_freq) + + else: # f ist list of array + df = f[1]-f[0] + + # if df is smaller than FSS bin width, do not integrate but just return interpolation + if df < 2e6: + if centers: + x = f - mix_freq + else: + x = f[0:-1]+0.5*df - mix_freq + eff = interp_eff(x) + eff_error_down = interp_eff_error_down(x) + eff_error_up = interp_eff_error_up(x) + eff_err = [eff_error_down, eff_error_up] + + # else pseudo integrate + else: + if centers: + M = len(f) + x = np.zeros((M, N)) + bin_centers = f-mix_freq + else: + M = len(f)-1 + x = x = np.zeros((M, N)) + bin_centers = f[0:-1]+0.5*df-mix_freq + + for i in range(M): + x[i] = np.linspace(bin_centers[i]-df/2, bin_centers[i]+df/2, N) + + + eff = interp_eff(x) + eff_err = [interp_eff_error_down(x), interp_eff_error_up(x)] + + + eff = np.sum(eff, axis=1)/N + #eff_err = [np.sqrt(np.sum(eff_err[0]**2, axis=1))/np.sqrt(N), + # np.sqrt(np.sum(eff_err[1]**2, axis=1))/np.sqrt(N)] + eff_err = [np.mean(eff_err[0], axis=1), np.mean(eff_err[1], axis=1)] + + + return eff, eff_err + + + +def pseudo_integrated_efficiency(f, df, snr_efficiency_dict, alpha=1): + if isinstance(f, float): + y, y_error = integrated_efficiency(f, snr_efficiency_dict, df) + pseudo_y = np.random.randn()*y_error# + y + else: + y, y_error = integrated_efficiency(f, snr_efficiency_dict, df) + + pseudo_y = np.random.randn(len(f)) + #print(pseudo_y) + pseudo_y[pseudo_y<0]*=y_error[0][pseudo_y<0] + pseudo_y[pseudo_y>0]*=y_error[1][pseudo_y>0] + #print(pseudo_y) + return pseudo_y+y, y_error + + +def binomial_interval(n, k, alpha=1 / 3.1514872): + + if isinstance(n, list) or type(n) is np.ndarray: + + mean = np.zeros(len(n)) + interval = np.zeros((2, len(n))) + interval_mean = np.zeros(len(n)) + for i in range(len(n)): + interval[0][i], interval[1][i] = proportion_confint(k[i], n[i], method='wilson', alpha=alpha) + rv = binom(n[i], k[i]*1./n[i]) + mean[i] = rv.mean()*1./n[i] + interval_mean[i] = np.mean(interval.T[i]) + interval_error = [interval_mean - interval[0], interval[1]-interval_mean] + + else: + l_int, u_int = proportion_confint(k, n, method='wilson', alpha=alpha) + rv = binom(n, k*1./n) + mean = rv.mean()*1./n + interval_mean = np.mean([l_int, u_int]) + interval_error = [interval_mean - l_int, u_int-interval_mean] + return mean, interval_mean, interval_error \ No newline at end of file diff --git a/mermithid/misc/__init__.py b/mermithid/misc/__init__.py index afb4ebbe..8654ab4e 100644 --- a/mermithid/misc/__init__.py +++ b/mermithid/misc/__init__.py @@ -8,3 +8,4 @@ from . import TritiumFormFactor from . import FakeTritiumDataFunctions from . import ConversionFunctions +from . import DetectionEfficiencyUtilities diff --git a/mermithid/processors/Fitters/MCUncertaintyPropagation.py b/mermithid/processors/Fitters/MCUncertaintyPropagation.py new file mode 100644 index 00000000..1a882dd2 --- /dev/null +++ b/mermithid/processors/Fitters/MCUncertaintyPropagation.py @@ -0,0 +1,150 @@ +''' +Author: C. Claessens +Date:8/2/2021 +Description: + + +''' + +from __future__ import absolute_import + +import numpy as np + +from morpho.utilities import morphologging, reader +from morpho.processors import BaseProcessor + +logger = morphologging.getLogger(__name__) + + + +__all__ = [] +__all__.append(__name__) + +class MCUncertaintyPropagation(BaseProcessor): + ''' + Processor that + Args: + + Inputs: + data: + Output: + result: dictionary containing fit results and uncertainties + ''' + def InternalConfigure(self, params): + ''' + Configure + ''' + + self.model = reader.read_param(params, 'model', "required") + self.fit = reader.read_param(params, 'fit_function', "required") + self.gen_and_fit = reader.read_param(params, 'gen_and_fit_function', "required") + self.fit_config_dict = reader.read_param(params, 'fit_config_dict', "required") + self.fit_options = reader.read_param(params, 'fit_options', "optional") + self.sample_parameters = reader.read_param(params, 'sample_parameters', []) + self.stat_sys_combined = reader.read_param(params, 'stat_sys_combined', [True, True, True]) + self.N = reader.read_param(params, 'N_samples', 50) + + return True + + def InternalRun(self): + + self.results = {} + + self.InitialFit() + self.ParameterSampling() + + #self.results['best_fit'] = list(self.fitted_params) + + return True + + def InitialFit(self): + ''' + Fit data with best model + + Returns + ------- + None. + + ''' + + self.fitted_params, self.fitted_params_errors, self.Counts = self.fit(self.data, + self.fit_config_dict, + self.fit_options) + + + logger.info('Best fit: {}'.format(self.fitted_params)) + x, pdf, bins, fitted_model, asimov_data = self.model(self.fit_config_dict, + self.fit_options, + params=self.fitted_params) + return True + + def ParameterSampling(self): + + start_j = 0 + + # propagate errors by sampling events and prior + return_dict_keys = ['stat', 'sys', 'combined'] + parameter_sampling = [{}, self.sample_parameters, self.sample_parameters] + + for k_i, k in enumerate(return_dict_keys): + if self.stat_sys_combined[k_i]: + + if k == 'sys': + fixed_data = ['asimov'] + elif k == 'combined': + fixed_data = [] + else: + fixed_data = [] + + + Nparams = len(self.fitted_params) + offsets = np.zeros((Nparams, self.N)) + sigmas = np.zeros((Nparams, self.N)) + fit_results = np.zeros((Nparams, self.N)) + parameter_samples = [] + + + # sequential sampling + all_fit_returns = [] + for i in range(start_j, self.N): + all_fit_returns.append(self.gen_and_fit(self.fitted_params, self.Counts, + self.fit_config_dict, + self.fit_options, + parameter_sampling[k_i], + i, fixed_data)) + + + + ################################### + # sort fit results + ################################### + #all_fit_returns = np.array(all_fit_returns) + for ii in range(len(all_fit_returns)): + j = ii + start_j + results = all_fit_returns[ii][0] + errors = all_fit_returns[ii][1] + parameter_samples.append(all_fit_returns[ii][2]) + + + for i in range(len(self.fitted_params)): + offsets[i][j] = results[i]-self.fitted_params[i] + sigmas[i][j] = errors[i] + fit_results[i][j] = results[i] + + + parameter_samples_transpose = {} + for p_key in parameter_samples[0].keys(): + parameter_samples_transpose[p_key] = [] + for p_i in parameter_samples: + parameter_samples_transpose[p_key].append(p_i[p_key]) + + + self.results[k] = {'offsets': [list(o) for o in offsets], + 'sigmas': [list(s) for s in sigmas], + 'results': [list(fr) for fr in fit_results], + 'parameter_samples': parameter_samples_transpose, + 'best_fit': list(self.fitted_params) + } + + + diff --git a/mermithid/processors/Fitters/__init__.py b/mermithid/processors/Fitters/__init__.py index 634545ab..40268845 100644 --- a/mermithid/processors/Fitters/__init__.py +++ b/mermithid/processors/Fitters/__init__.py @@ -4,4 +4,5 @@ from __future__ import absolute_import from .BinnedDataFitter import BinnedDataFitter +from .MCUncertaintyPropagation import MCUncertaintyPropagation diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py new file mode 100755 index 00000000..659192f8 --- /dev/null +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -0,0 +1,1289 @@ + +""" +Author: C. Claessens +Date:8/03/2021 +Description: + Tritium class + + contains tritium model(s) + convolves with energy resolution + convolves with lineshape + multiplies with efficiency + adds background + has log likelihood and neg log likelihood functions (poisson) that can be used by minimzer + + +""" + +import numpy as np +from scipy import constants +from scipy.special import erfc +import json +import os +from copy import deepcopy +import matplotlib.pyplot as plt + + +from mermithid.processors.Fitters import BinnedDataFitter +from mermithid.misc.FakeTritiumDataFunctions import * +from mermithid.processors.misc.KrComplexLineShape import KrComplexLineShape + +from morpho.utilities import morphologging, reader +logger = morphologging.getLogger(__name__) + +#import helper_functions.plot_methods as pm +from mermithid.misc.DetectionEfficiencyUtilities import * +#import importlib +#importlib.reload(det_eff) + +electron_mass = constants.electron_mass/constants.e*constants.c**2 +FineStructureConstant = 0.0072973525664 + +# # for converting numpy array to double +# def float2double(a): +# """ +# convert floats (or array of floats) to double +# """ +# if a is None or a.dtype == np.float64: +# return a +# else: +# return a.astype(np.float64) + + +class BinnedTritiumMLFitter(BinnedDataFitter): + + + def InternalConfigure(self, config_dict): + + # model options + self.use_approx_model = reader.read_param(config_dict, 'use_approximate_model', True) + self.use_toy_model_efficiency = reader.read_param(config_dict, 'use_toy_model_efficiency', False) + self.fit_nu_mass = reader.read_param(config_dict, 'fit_neutrino_mass', False) + self.is_distorted = reader.read_param(config_dict, 'distorted', False) # multiply spectrum with efficency + self.is_smeared = reader.read_param(config_dict, 'smeared', True) # convolve with energy resolution + self.error_scaling = reader.read_param(config_dict, 'error_scaling', 1.) # scales efficiency uncertainty + self.is_scattered = reader.read_param(config_dict, 'scattered', False) # convolve with lineshape + self.integrate_bins = reader.read_param(config_dict, 'integrate_bins', True) # integrate spectrum over bin widths + self.fit_efficiency_tilt = reader.read_param(config_dict, 'fit_efficiency_tilt', False) # efficiency slope is free parameter + + + # save plots in + self.savepath = reader.read_param(config_dict, 'plot_path', '.') + + + # model parameters and uncertainties + self.B_mean = reader.read_param(config_dict, 'B_mean', 1) + self.B_width = reader.read_param(config_dict, 'B_width', 1e-6) + + # efficiency + self.efficiency_file_path = reader.read_param(config_dict, 'efficiency_file_path', '') + + + # final state spectrum + self.use_final_states = reader.read_param(config_dict, 'use_final_states', False) + self.final_state_array = reader.read_param(config_dict, 'final_state_array', [[0], [1]]) + + + # detector response + self.NScatters = reader.read_param(config_dict, 'NScatters', 20) + self.resolution_model = reader.read_param(config_dict, 'resolution_model', 'gaussian') + self.simplified_lineshape_path = reader.read_param(config_dict, 'simplified_lineshape_path', "required") + self.helium_lineshape_path = reader.read_param(config_dict, 'helium_lineshape_path', "optional") + self.hydrogen_proportion = reader.read_param(config_dict, 'hydrogen_proportion', 1) + self.use_helium_scattering = reader.read_param(config_dict, 'use_helium_scattering', False) + + self.scatter_peak_ratio_b_mean = reader.read_param(config_dict, 'scatter_peak_ratio_b_mean', 0.7) + self.scatter_peak_ratio_b_width = reader.read_param(config_dict, 'scatter_peak_ratio_b_width', 0.1) + self.scatter_peak_ratio_c_mean = reader.read_param(config_dict, 'scatter_peak_ratio_c_mean', 0.7) + self.scatter_peak_ratio_c_width = reader.read_param(config_dict, 'scatter_peak_ratio_c_width', 0.1) + + self.survival_mean = reader.read_param(config_dict, 'survival_mean', 0.5) + self.survival_error = reader.read_param(config_dict, 'survival_width', 0.1) + + self.res_mean = reader.read_param(config_dict, 'gaussia_resolution_mean', 15.0) + self.res_width = reader.read_param(config_dict, 'gaussian_resolution_width', 1.0) + + self.two_gaussian_mu1 = reader.read_param(config_dict, 'two_gaussian_mu1', 0) + self.two_gaussian_mu2 = reader.read_param(config_dict, 'two_gaussian_mu2', 0) + self.two_gaussian_sig1_mean = reader.read_param(config_dict, 'two_gaussian_sig1_mean', 0) + self.two_gaussian_sig2_mean = reader.read_param(config_dict, 'two_gaussian_sig2_mean', 0) + self.two_gaussian_sig1_width = reader.read_param(config_dict, 'two_gaussian_sig1_width', 15) + self.two_gaussian_sig2_width = reader.read_param(config_dict, 'two_gaussian_sig2_width', 0) + self.two_gaussian_wide_fraction = reader.read_param(config_dict, 'wide_gaussian_weigth', 1.) + + + ######################### + # fit configuration + ######################## + self.counts_guess = reader.read_param(config_dict, 'counts_guess', 5000) + self.fix_counts = False + self.mass_guess = reader.read_param(config_dict, 'nu_mass_guess', 0.0) + + # frequency range + self.min_frequency = reader.read_param(config_dict, 'min_frequency', "required") + self.max_frequency = reader.read_param(config_dict, 'max_frequency', None) + + + # initial values + self.B = self.B_mean + self.res = self.res_mean + self.scatter_ratio = self.survival_mean + self.endpoint=config_dict['true_endpoint'] + + # some defaults + self.use_asimov = False + + + + ######################################### + # detection efficiency specification + ######################################### + + self.tilted_efficiency = False + self.tilt = 0. + + if self.use_toy_model_efficiency: + x = np.linspace(self.min_frequency, self.max_frequency, 100) + y = power_efficiency(x, plot=False) + + self.power_eff_interp = interp1d(x, y[0], fill_value=0, bounds_error=False) + self.power_eff_error_interp = interp1d(x, y[1][0], fill_value=1, bounds_error=False) + + elif self.efficiency_file_path != '': + with open(self.efficiency_file_path, 'r') as infile: + snr_efficiency_dict = json.load(infile) + + # don't ask ... it's complicated + snr_efficiency_dict['frequency'] = snr_efficiency_dict['frequencies'] + snr_efficiency_dict['good_fit_index'] = [True]*len((snr_efficiency_dict['frequencies'])) + snr_efficiency_dict['tritium_rates'] = snr_efficiency_dict['eff interp with slope correction'] + snr_efficiency_dict['tritium_rates_error'] = snr_efficiency_dict['error interp with slope correction'] + + if self.max_frequency == None: + self.max_frequency = np.max(snr_efficiency_dict['frequency']) + + self.snr_efficiency_dict = snr_efficiency_dict + else: + self.efficiency_file_path = '' + + if self.max_frequency == None: + raise ValueError('Max frequency undetermined') + + ######################################### + # lineshape specification + ######################################### + + # simplified lineshape parameters + self.lineshape_p = np.loadtxt(self.simplified_lineshape_path, unpack=True) + + # if true lineshape is plotted during tritium spectrum shape generation + self.plot_lineshape = False + + # helium lineshape + if self.use_helium_scattering: + self.helium_lineshape_p = np.loadtxt(self.helium_lineshape_path, unpack=True) + + # which lineshape should be used? + self.lineshape_model = reader.read_param(config_dict, 'which_lineshape', 'two_gaussian') + if self.lineshape_model == 'simplified': + self.multi_gas_lineshape = self.simplified_multi_gas_lineshape + elif self.lineshape_model == 'accurate_simplified': + self.multi_gas_lineshape = self.more_accurate_simplified_multi_gas_lineshape + + elif self.lineshape_model =='detailed': + self.detailed_scatter_spectra_path = reader.read_param(config_dict, 'detailed_lineshape_path') + + ## lineshape params + #self.SimpParams = [self.res*2*np.sqrt(2*np.log(2)), self.scatter_ratio] + + # Setup and configure lineshape processor + complexLineShape_config = { + 'gases': ["H2","He"], + 'max_scatters': self.NScatters, + 'fix_scatter_proportion': True, + # When fix_scatter_proportion is True, set the scatter proportion for gas1 below + 'gas1_scatter_proportion': self.hydrogen_proportion, + # This is an important parameter which determines how finely resolved + # the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to + # be increased for larger datasets. + 'num_points_in_std_array': 10000, + 'B_field': self.B, + 'base_shape': 'dirac', + 'path_to_osc_strengths_files': self.detailed_scatter_spectra_path + } + logger.info('Setting up complex lineshape object') + self.complexLineShape = KrComplexLineShape("complexLineShape") + logger.info('Configuring complex lineshape') + self.complexLineShape.Configure(complexLineShape_config) + logger.info('Checking existence of scatter spectra files') + self.complexLineShape.check_existence_of_scatter_file() + + self.multi_gas_lineshape = self.complex_lineshape + else: + self.multi_gas_lineshape = self.more_accurate_simplified_multi_gas_lineshape + logger.info('Using default lineshape: more-accurate-simplified') + + # scatter peak ratio + self.scatter_peak_ratio = reader.read_param(config_dict, 'scatter_peak_ratio', 'modified_exponential') + + if self.scatter_peak_ratio == 'constant': + self.use_fixed_scatter_peak_ratio = True + elif self.scatter_peak_ratio == 'modified_exponential': + self.use_fixed_scatter_peak_ratio = False + + else: + logger.error("Configuration of scatter_peak_ratio not known. Options are 'constant' and 'modifed_exponential") + raise ValueError("Configuration of scatter_peak_ratio not known. Options are 'constant' and 'efficiency_model") + + + ######################################### + # energies and bins + ######################################### + + # internal variables + self.dbins = reader.read_param(config_dict, 'energy_bin_width', 50) + self.denergy = reader.read_param(config_dict, 'energy_step_size', min([np.round(self.dbins/10, 2), 1])) + + # bin size has to divide energy step size + self.N_bins = np.round((self.Energy(self.min_frequency)-self.Energy(self.max_frequency))/self.dbins) + self.N_energy_bins = self.N_bins*np.round(self.dbins/self.denergy) + + # adjust energy stepsize to match bin division + self.denergy = self.dbins/np.round(self.dbins/self.denergy) + + self._energies = np.arange(self.Energy(self.max_frequency), self.Energy(self.max_frequency)+(self.N_energy_bins)*self.denergy, self.denergy) + self._bins = np.arange(np.min(self.energies), self.Energy(self.max_frequency)+(self.N_bins)*self.dbins, self.dbins) + + if len(self._energies) > self.N_energy_bins: + self._energies = self._energies[:-1] + if len(self._bins) > self.N_bins: + self._bins = self._bins[:-1] + + #print(len(self._energies), self.N_energy_bins) + #print(len(self._bins), self.N_bins) + #print(self.dbins/self.denergy) + + + + self.bin_centers = self._bins[0:-1]+0.5*(self._bins[1]-self._bins[0]) + self.freq_bins = self.Frequency(self._bins) + self.freq_bin_centers = self.Frequency(self.bin_centers) + + #self._hist = [] + #self._data = [] + + self._bin_efficiency, self._bin_efficiency_errors = [], [] + self._full_efficiency, self._full_efficiency_errors = [], [] + self._bin_efficiency, self._bin_efficiency_errors = self.Efficiency(self.bin_centers) + self._full_efficiency, self._full_efficiency_errors = self.Efficiency(self.energies) + + + ######################################### + # configure parent BinnedDataFitter + ######################################### + + # overwrite model + self.model = self.TritiumSpectrumBackground + + + # configure fit + energy_limits = [max([self.endpoint-300, np.min(self.energies)+3.5]), min([self.endpoint+300, np.max(self.energies)-3.5])] + neutrino_limits = [-(np.max(self.energies) - energy_limits[1]-1)**2, (energy_limits[0]-np.min(self.energies)-1)**2] + + + #logger.warning('Neutrino mass fitted: {}'.format(self.fit_nu_mass)) + if not self.fit_efficiency_tilt: + self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', 'scatter_peak_ratio_b', 'scatter_peak_ratio_c'] + self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_c_mean] + self.fixes = [False, False, not self.fit_nu_mass, self.fix_counts, True, True] + self.limits = [energy_limits, + [1e-10, None], + neutrino_limits, + [0, None], + [0.1, 1.], + [0.1, 1.]] + + + + self.parameter_errors = [max([0.1, 0.1*p]) for p in self.initial_values] + + + else: + logger.warning('Efficiency tilt will be fitted') + self.tilted_efficiency = True + self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', 'Efficiency tilt'] + self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_c_mean, self.tilt] + self.parameter_errors = [max([0.1, 0.1*p]) for p in self.initial_values] + self.fixes = [False, False, not self.fit_nu_mas, self.fix_counts, True, True, False] + self.limits = [energy_limits, + [1e-10, None], + neutrino_limits, + [0, None], + [0.1, 1.], + [0.1, 1.], + [-0.5, 0.5]] + + + self.constrained_parameters = reader.read_param(config_dict, 'constrained_parameters', []) + if len(self.constrained_parameters) > 0: + logger.warning('Some parameters are constrained') + self.constrained_means = reader.read_param(config_dict, 'constrained_means', []) + self.constrained_widths = reader.read_param(config_dict, 'constrained_widths', []) + self.print_level = 0 + + + return True + + + # parameter sampling + def Gaussian_sample(self, mean, width): + np.random.seed() + return np.random.randn()*width+mean + + def Beta_sample(self, mean, width): + a = ((1-mean)/(width**2)-1/mean)*mean**2 + b = (1/mean-1)*a + return np.random.beta(a, b) + + + ############################ conversion methods ########################### + + def Energy(self, f, mixfreq=0.): + """ + converts frequency in Hz to energy in eV + """ + emass = constants.electron_mass/constants.e*constants.c**2 + gamma = (constants.e*self.B)/(2.0*np.pi*constants.electron_mass) * 1./(f+mixfreq) + + return (gamma -1.)*emass + + + def Frequency(self, E, Theta=None): + """ + converts energy in eV to frequency in Hz + """ + if Theta==None: + Theta=np.pi/2 + + emass = constants.electron_mass/constants.e*constants.c**2 + gamma = E/(emass)+1. + + return (constants.e*self.B)/(2.0*np.pi*constants.electron_mass) * 1./gamma + + + + ################################# bins and data ######################################## + + + @property + def energies(self): + return self._energies + + @energies.setter + def energies(self, some_energies): + """ + fine grained array. + delete pre-existing efficiencies when resetting energies. + """ + self._energies = some_energies + + self._full_efficiency, self._full_efficiency_errors = [], [] + efficiency = self.Efficiency(self._energies) + self._full_efficiency = efficiency[0] + self._full_efficiency_errors = efficiency[1] + + """super(Tritium, self).__init__(a=np.min(self._energies), + b=np.max(self._energies), + xtol=self.xtol, seed=self.seed)""" + + + @property + def bins(self): + return self._bins#, self._freq_bin_centers + + @bins.setter + def bins(self, some_bins): + """ + energy bins. + delete pre-existing efficiencies when resetting bins. + """ + + # energy bins + self._bins = some_bins + self.bin_centers = self._bins[0:-1] +0.5*(self._bins[1]-self._bins[0]) + + # frequency bins + self.freq_bins = self.Frequency(self._bins) + self.freq_bin_centers = self.Frequency(self.bin_centers) + + # bin efficiencies + self._bin_efficiency, self._bin_efficiency_errors = [], [] + self._bin_efficiency, self._bin_efficiency_errors = self.Efficiency(self.freq_bin_centers, freq=True) + + + def ReSetBins(self): + #self.energies = np.arange(self.Energy(self.max_frequency), self.Energy(self.min_frequency), self.denergy) + #self.bins = np.arange(np.min(self.energies), np.max(self.energies), self.dbins) + + self.energies = np.arange(self.Energy(self.max_frequency), self.Energy(self.max_frequency)+(self.N_energy_bins)*self.denergy, self.denergy) + self.bins = np.arange(np.min(self.energies), self.Energy(self.max_frequency)+(self.N_bins)*self.dbins, self.dbins) + self._bin_efficiency, self._bin_efficiency_error = [], [] + + if len(self._energies) > self.N_energy_bins: + self._energies = self._energies[:-1] + if len(self._bins) > self.N_bins: + self._bins = self._bins[:-1] + + + + def GenerateData(self, params, N): + + #print('Generating data') + x = self.energies[0:-1]+0.5*(self.energies[1]-self.energies[0]) + pdf = np.longdouble(self.TritiumSpectrumBackground(x, *params)) + pdf[pdf<0]=0. + np.random.seed() + + pdf = np.float64(np.longdouble(pdf/np.sum(pdf))) + + + self.data = np.random.choice(x, np.random.poisson(N), p=pdf/np.sum(pdf)) + self.freq_data = self.Frequency(self.data) + + return self.data, self.freq_data + + + def GenerateAsimovData(self, params): + + asimov_data = [] + #x = self.bins #np.arange(min(self.energies), max(self.energies), 25) + #xc = x[0:-1]+0.5*(x[1]-x[0]) + #pdf = np.round(self.TritiumSpectrumBackground(xc, *params)) + + self.use_asimov = True + #for i, p in enumerate(pdf): + # asimov_data.extend(list(itertools.repeat(xc[i], int(p)))) + + return np.array(asimov_data), self.Frequency(np.array(asimov_data)) + + + + def ConvertFreqData2EnergyData(self): + """ + Convert frequencies to energyies with set B + """ + self.data = self.Energy(self.freq_data) + return self.data + + + def Histogram(self, weights=None): + """ + histogram data using bins + """ + h, b = np.histogram(self.data, bins=self.bins, weights=weights) + self.hist = h#float2double(h) + return h + + + def ConvertAndHistogram(self, weights=None): + """ + histogram data using bins + """ + self.data = self.ConvertFreqData2EnergyData() + self.hist, b = np.histogram(self.data, bins=self.bins, weights=weights) + #self.hist = float2double(h) + return self.hist + + + def SamplePriors(self, sampled_parameters): + self.parameter_samples = {} + sample_values = [] + if 'res' in sampled_parameters.keys(): + self.res = self.Gaussian_sample(self.res_mean, self.res_width) + self.parameter_samples['res'] = self.res + sample_values.append(self.res) + if 'scatter_peak_ratio_b' in sampled_parameters.keys(): + self.scatter_peak_ratio_b = self.Beta_sample(self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_b_width) + self.parameter_samples['scatter_peak_ratio_b'] = self.scatter_peak_ratio_b + sample_values.append(self.scatter_peak_ratio_b) + if 'scatter_peak_ratio_c' in sampled_parameters.keys(): + self.scatter_peak_ratio_c = self.Beta_sample(self.scatter_peak_ratio_c_mean, self.scatter_peak_ratio_c_width) + self.parameter_samples['scatter_peak_ratio_c'] = self.scatter_peak_ratio_c + sample_values.append(self.scatter_peak_ratio_c) + if 'B' in sampled_parameters.keys(): + self.B = self.Gaussian_sample(self.B_mean, self.B_width) + self.parameter_samples['B'] = self.B + sample_values.append(self.B) + + logger.info('Samples are: {}'.format(sample_values)) + # set new values in model + self.initial_values[4:6] = [self.scatter_peak_ratio_b, self.scatter_peak_ratio_c] + + return self.parameter_samples + + def SampleConvertAndFit(self, sampled_parameters={}, params= []): + + if self.use_asimov: + #temp = self.error_scaling + #self.error_scaling = 0 + #self._bin_efficiency, self._bin_efficiency_error = self.Efficiency(self.bin_centers, pseudo=True) + self.hist = self.TritiumSpectrumBackground(self.bin_centers, *params) + #self.error_scaling = temp + # if random_priors contains 3 items (all boolean) get new sample froom priors + if len(sampled_parameters.keys()) > 0: + logger.info('Sampling: {}'.format(sampled_parameters.keys())) + self.SamplePriors(sampled_parameters) + # re-calculate bin efficiencies, if self.pseudo_eff=True efficiency will be ranomized + + # need to first re-calcualte energy bins with sampled B before getting efficiency + self.ReSetBins() + + if 'efficiency' in sampled_parameters.keys(): + random_efficiency = True + else: + random_efficiency = False + self._bin_efficiency, self._bin_efficiency_error = self.Efficiency(self.bin_centers, pseudo=random_efficiency) + + + # now convert frequency data to energy data and histogram it + if not self.use_asimov: + self.ConvertAndHistogram() + + + + return self.fit() + + + + + ########################### Tritium spectrum ################################# + + + def gauss_resolution_f(self, energy_array, A, sigma, mu): + f = A*(1/(sigma*np.sqrt(2*np.pi)))*np.exp(-(((energy_array-mu)/sigma)**2.)/2.) + return f + + + def approximate_shape(self, K, Q, m_nu, index): + """ m_nu is neutrino mass squared """ + shape = np.zeros(len(K)) + nu_mass_shape = ((Q - K[index])**2 -m_nu)**0.5 + shape[index] = (Q - K[index])*nu_mass_shape + return shape + + + def approximate_spectrum(self, E, Q, m_nu=0): + """ + model as in mermithid fake data generator: + https://github.com/project8/mermithid/blob/feature/phase2-analysis/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py + + but the ephasespace is approximate (some factors neglected) + """ + mnu = np.real(m_nu**0.5) + + if self.use_final_states: + if isinstance(E, list) or isinstance(E, np.ndarray): + N_states = len(self.final_state_array[0]) + Q_states = Q+self.final_state_array[0]-np.max(self.final_state_array[0]) + approximate_e_phase_space = self.ephasespace(E, Q) + + + index = [np.where(E < Q_states[i]-mnu) for i in range(N_states)] + beta_rates_array = [self.approximate_shape(E, Q_states[i], m_nu, index[i]) + * self.final_state_array[1][i] + * approximate_e_phase_space for i in range(N_states)] + + to_return = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) + return to_return + + else: + logger.warning('E is not array!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') + + return_value = 0. + + for i, e_binding in enumerate(self.final_state_array[0]): + # binding energies are negative + Q_state = Q+e_binding + if Q_state-mnu > E > 0: + return_value += self.final_state_array[1][i] *(GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.ephasespace(E, Q_state)* + (Q_state - E)*np.sqrt((Q_state - E)**2 - (mnu)**2)) + + return return_value/np.sum(self.final_state_array[1]) + + else: + beta_rates = np.zeros(len(E)) + + index = np.where(E < Q-mnu) + K = E[index] + + nu_mass_shape = ((Q - K)**2 -m_nu)**0.5 + beta_rates[index] = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.ephasespace(K, Q)*(Q - K)*nu_mass_shape + + return beta_rates + + def ephasespace(self, K, Q): + #G = rad_corr(K, Q) #Radiative correction + #S = screen_corr(K) #Screening factor + #I = exchange_corr(K) #Exchange correction + #R = recoil_corr(K, Q) #Recoil effects + #LC = finite_nuc_corr(K, Q) #Finite nucleus corrections + #X = coul_corr(K, Q) #Recoiling Coulomb field correction + F = fermi_func(K) #Uncorrected Fermi function + return pe(K)*Ee(K)*F#*G*S*I*R*LC*X + + def which_model(self, *pars): + if self.use_approx_model: + return self.approximate_spectrum(*pars) + else: + return self.effective_TritiumSpectrumShape(*pars) + + def scatter_peak_ratio(self, prob_b, prob_c, j): + ''' + ratio of successive peaks taking reconstruction efficiency into account + ''' + return np.exp(-prob_b*j**prob_c) + + + def simplified_ls(self, K, Kcenter, FWHM, prob_b, prob_c=1): + """ + Simplified lineshape. sum of Gaussians imitating hydrogen only lineshape + """ + + p0, p1, p2, p3 = self.lineshape_p[1], self.lineshape_p[3], self.lineshape_p[5], self.lineshape_p[7] + sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) + shape = gaussian(K, [sig0, Kcenter]) + norm = 1. + + for i in range(1, self.NScatters): + sig = p0[i]+p1[i]*FWHM + mu = -(p2[i]+p3[i]*np.log(FWHM-30)) + + if self.use_fixed_scatter_peak_ratio: + probi = prob_b**(i+1) + else: + probi = self.scatter_peak_ratio(prob_b, prob_c, i) + + shape += probi*gaussian(K, [sig, mu+Kcenter]) + norm += probi + + return shape, norm + + def f_i(self, K, gamma, mu, sigma): + z = (mu+gamma*sigma**2+K)/(np.sqrt(2)*sigma) + f = np.exp(gamma*(mu+K+gamma*sigma**2/2.))*erfc(z) + return f/np.sum(f) + + def more_accurate_simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): + """ + Still a simplified lineshape but helium is not just a gaussian + """ + if self.plot_lineshape: + logger.info('Using more accurate multi gas scattering. Hydrogen proportion is {}'.format(self.hydrogen_proportion)) + + + p0, p1, p2, p3 = self.lineshape_p[1], self.lineshape_p[3], self.lineshape_p[5], self.lineshape_p[7] + q0, q1, q2, q3, q4, q5, q6, q7 = self.helium_lineshape_p[1], self.helium_lineshape_p[3],\ + self.helium_lineshape_p[5], self.helium_lineshape_p[7],\ + self.helium_lineshape_p[9], self.helium_lineshape_p[11],\ + self.helium_lineshape_p[13], self.helium_lineshape_p[15] + + + + sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) + shape0 = gaussian(K, [sig0, Kcenter]) + shape0 *= 1/np.sum(shape0) + norm = 1. + norm_h = 1. + norm_h2 = 1. + + hydrogen_scattering = np.zeros(len(K)) + helium_scattering = np.zeros(len(K)) + + #plt.figure(figsize=(10,10)) + + for i in range(1, self.NScatters): + + # hydrogen scattering + sig = p0[i]+p1[i]*FWHM + mu = -(p2[i]+p3[i]*np.log(FWHM-30)) + + if self.use_fixed_scatter_peak_ratio: + probi = prob_b**(i) + else: + probi = self.scatter_peak_ratio(prob_b, prob_c, i) + + h_scatter_i = gaussian(K, [sig, mu+Kcenter]) + hydrogen_scattering += probi*h_scatter_i/np.sum(h_scatter_i) + norm += probi + #plt.plot(K, h_scatter_i, color='blue', label='hydrogen') + + # helium scattering + mu_he = (q0[i]+q1[i]*FWHM+q2[i]*FWHM**2) + sig_he = q3[i]+q4[i]*FWHM + gamma_he = q5[i]+q6[i]*FWHM+q7[i]*FWHM**2 + he_scatter_i = self.f_i(K, gamma_he, mu_he, sig_he) + helium_scattering += probi * he_scatter_i + + #plt.plot(K, (he_scatter_i/(np.sum(he_scatter_i)*(K[1]-K[0]))), color='cyan') + + #plt.plot(K, (shape0 + hydrogen_scattering)/np.max(shape0 + hydrogen_scattering), color='blue', label='hydrogen') + #plt.plot(K, (shape0 + helium_scattering)/np.max(shape0 + helium_scattering), color='red', label='helium') + + # full lineshape + norm_h = np.sum(shape0 + hydrogen_scattering) + norm_he = np.sum(shape0 + helium_scattering) + + lineshape = (self.hydrogen_proportion*(shape0 + hydrogen_scattering)/norm_h + + (1-self.hydrogen_proportion)*(shape0 + helium_scattering)/norm_he) + + #plt.plot(K, lineshape/np.max(lineshape), color='darkgreen', label='full: {} hydrogen'.format(self.hydrogen_proportion)) + #plt.xlim(-200, 200) + #plt.legend() + + return lineshape, norm + + def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): + """ + This uses Gaussians of different mu and sigma for different gases + """ + if self.plot_lineshape: + logger.info('Using gaussian multi gas scattering') + + p0, p1, p2, p3 = self.lineshape_p[1], self.lineshape_p[3], self.lineshape_p[5], self.lineshape_p[7] + q0, q1, q2, q3 = self.helium_lineshape_p[1], self.helium_lineshape_p[3], self.helium_lineshape_p[5], self.helium_lineshape_p[7] + + + sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) + shape = gaussian(K, [sig0, Kcenter]) + norm = 1. + + hydrogen_scattering = np.zeros(len(K)) + helium_scattering = np.zeros(len(K)) + + #plt.figure(figsize=(10,10)) + + for i in range(i, self.NScatters): + + # hydrogen scattering + sig = p0[i]+p1[i]*FWHM + mu = -(p2[i]+p3[i]*np.log(FWHM-30)) + + if self.use_fixed_scatter_peak_ratio: + probi = prob_b**(i) + else: + probi = self.scatter_peak_ratio(prob_b, prob_c, i) + + h_scatter_i = probi*gaussian(K, [sig, mu+Kcenter]) + hydrogen_scattering += h_scatter_i + norm += probi + #plt.plot(K, h_scatter_i, color='blue', label='hydrogen') + + # helium scattering + mu_he = -(q0[i]+q1[i]*FWHM) + sig_he = q2[i]+q3[i]*FWHM + he_scatter_i = probi*self.gauss_resolution_f(K, 1, sig_he, mu_he+Kcenter) + helium_scattering += he_scatter_i + + #plt.plot(K, he_scatter_i, color='red', label='helium') + + #plt.plot(K, (shape + hydrogen_scattering)/np.max(shape + hydrogen_scattering), color='blue', label='hydrogen') + #plt.plot(K, (shape + helium_scattering)/np.max(shape + helium_scattering), color='red', label='helium') + # full lineshape + lineshape = (shape + self.hydrogen_proportion*hydrogen_scattering + (1-self.hydrogen_proportion)*helium_scattering) + + #plt.plot(K, lineshape/np.max(lineshape), color='black', label='full') + #plt.xlim(-200, 200) + #plt.legend() + + return lineshape, norm + + """def complex_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): + lineshape_rates = self.complexLineShape.spectrum_func_1(K/1000., FWHM, 0, 1, prob_b) + plt.plot(K, lineshape_rates/np.max(lineshape_rates), label='complex lineshape', color='purple', linestyle='--') + return lineshape_rates""" + + + def running_mean(self, x, N): + N_round = round(N) + cumsum = np.cumsum(x) + return (cumsum[int(N_round)::int(N_round)] - cumsum[:-int(N_round):int(N_round)]) / float(N_round) + + + def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=None, tilt=0., error=False): + E = np.array(E) + + if len(E)==0: + E = self._bin_centers + + """############### E is float ############### + if isinstance(E, float): + if E+m_nu > endpoint: + K_spec = 0. + else: + + efficiency = 1. + if self.is_distorted == True: + efficiency = self.Efficiency(E)[0] + K_spec = self.which_model(E, endpoint, m_nu)*efficiency/self.norm""" + + + ################## E is list or array ################# + + + # smear spectrum + if self.is_smeared or self.is_scattered: + + + max_energy = 1000 + dE = self.energies[1]-self.energies[0]#E[1]-E[0] + n_dE = round(max_energy/dE) + #e_add = np.arange(np.min(self.energies)-round(max_energy/dE)*dE, np.min(self.energies), dE) + e_lineshape = np.arange(-n_dE*dE, n_dE*dE, dE) + + + e_spec = np.arange(min(self.energies)-max_energy, max(self.energies)+max_energy, dE) + #np.r_[e_add, self._energies] + + # energy resolution + if self.resolution_model != 'two_gaussian': + lineshape = self.gauss_resolution_f(e_lineshape, 1, self.res, 0) + else: + lineshape = self.wide_fraction * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_1, self.two_gaussian_mu1) + (1 - self.wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_2, self.two_gaussian_mu2) + + + # spectrum shape + spec = self.which_model(e_spec, endpoint, m_nu) + spec[np.where(e_spec>endpoint-np.abs(m_nu)**0.5)]=0. + + if not self.is_scattered: + # convolve with gauss with spectrum + K_convolved = convolve(spec, lineshape, mode='same') + below_Kmin = np.where(e_spec < min(self.energies)) + #np.put(K_convolved, below_Kmin, np.zeros(len(below_Kmin))) + K_convolved = np.interp(self.energies, e_spec, K_convolved) + #K_convolved = K_convolved[e_spec>=np.min(self.energies)] + + + if self.is_scattered: + if prob_b == None: + prob_b = self.scatter_peak_ratio_b_mean + prob_c = self.scatter_peak_ratio_c_mean + + + # simplified lineshape + + FWHM = 2.*np.sqrt(2.*np.log(2.))*self.res + + # sigmas = self.lineshape_p[1]+FWHM*self.lineshape_p[3] + # mus = self.lineshape_p[5]+self.lineshape_p[7]*np.log(FWHM-30) + # scatter_norm = 1 + + + + # if self.plot_lineshape: + # logger.info('Plotting lineshape') + # # start lineshape plot + # fig, ax = plt.subplots(1, 1, figsize=(7,5)) + # ax.plot(e_lineshape, lineshape, label='Unscattered resolution') + + + # for i in range(1, self.NScatters+1): + + + # gauss_i = self.gauss_resolution_f(e_lineshape, 1, sigmas[i-1], -mus[i-1]) + # lineshape += gauss_i*ratio**i + # scatter_norm += ratio**i + + # if self.plot_lineshape: + # ax.plot(e_lineshape, gauss_i*ratio**i, label='Scatter peak: {}'.format(i)) + + + # lineshape *= 1/scatter_norm + + + # get lineshape + if not self.include_helium_scattering: + tail, norm = self.simplified_ls(e_lineshape, 0, FWHM, prob_b, prob_c) + else: + tail, norm = self.multi_gas_lineshape(e_lineshape, 0, FWHM, prob_b, prob_c) + + lineshape += tail + lineshape = lineshape/norm + + # lineshape done now convolve + K_convolved = convolve(spec, lineshape, mode='same') + below_Kmin = np.where(e_spec < min(self.energies)) + #np.put(K_convolved, below_Kmin, np.zeros(len(below_Kmin))) + K_convolved = np.interp(self.energies, e_spec, K_convolved) + #K_convolved = K_convolved[e_spec>=np.min(self.energies)] + + if self.plot_lineshape: + #ax.plot(e_gauss, full_lineshape, label='Full lineshape', color='grey') + # print(FWHM, ratio) + # ax.set_xlabel('Energy [eV]') + # ax.set_ylabel('Amplitude') + # ax.legend(loc='best') + # plt.xlim(-500, 250) + # plt.tight_layout() + # plt.savefig(os.path.join(self.savepath, 'scattering_model.pdf'), dpi=200, transparent=True) + + plt.figure(figsize=(7,5)) + #plt.plot(e_lineshape, self.simplified_ls(e_lineshape, 0, FWHM, ratio), color='red', label='FDG') + plt.plot(e_lineshape, lineshape/np.max(lineshape), label = 'Full lineshape', color='Darkblue') + + simple_ls = self.simplified_ls(e_lineshape, 0, FWHM, ratio) + plt.plot(e_lineshape, simple_ls/np.max(simple_ls), label='Hydrogen only lineshape', color='red') + plt.xlabel('Energy [eV]') + plt.ylabel('Amplitude') + #plt.grid() + plt.xlim(-500, 250) + plt.legend(loc='best') + plt.tight_layout() + plt.savefig(os.path.join(self.savepath, 'lineshape.pdf'), dpi=200, transparent=True) + + + + else: + # base shape + spec=np.zeros(len(self._energies)) + index=np.where(self._energies<=endpoint-np.abs(m_nu)**0.5) + try: + tritium = self.which_model(self.energies[index], endpoint, m_nu) + except Exception as e: + print(E) + print(index) + print(endpoint) + print(m_nu) + print(self.is_smeared) + raise(e) + + spec[index]=tritium + K_convolved = spec + + # integrate bins + if self.integrate_bins: + + dE = self.denergy + dE_bins = E[1]-E[0] + N = np.round(dE_bins/dE,4) + + if not (np.abs(N%1) < 1e-6 or np.abs(N%1 - 1)<1e-6): + logger.error('N', N) + logger.error('modulo', N%1, N%1-1) + raise ValueError('bin sizes have to divide') + + + if N > 1: + #print('Integrate spectrum', len(K_convolved)) + if E[0] -0.5*dE_bins >= self._energies[1]: + K_convolved_cut = K_convolved[self.energies>=min(E[0] -0.5*dE_bins)] + K_convolved = self.running_mean(K_convolved_cut, N) + logger.warning('Cutting spectrum below Kmin:{} > {}'.format(E[0]-0.5*dE_bins, self_energies[0])) + else: + K_convolved = self.running_mean(K_convolved, N) + + else: + # till here K_convolved was defiend on self._energies + # now we need it on E + K_convolved = np.interp(E, self.energies, K_convolved)*(len(E)*1./len(self._energies)) + + + else: + #logger.warning('WARNING: tritium spectrum is not integrated over bin widths') + K_convolved = np.interp(E, self._energies, K_convolved)*(len(E)*1./len(self._energies)) + + + # multiply efficiency + efficiency = np.ones(len(E)) + efficiency_errors = [np.zeros(len(E)), np.zeros(len(E))] + + + if self.is_distorted == True: + if self.fit_efficiency_tilt: + self.tilt = tilt + efficiency, efficiency_errors = self.Efficiency(E) + + K_eff=K_convolved*efficiency + # finally + K = K_eff/np.sum(K_eff)*np.sum(K_convolved) + + # if error from efficiency on spectrum shape should be returned + if error: + K_error=np.zeros((2, len(E))) + K_error = K_convolved*(efficiency_errors)/np.sum(K_eff)*np.sum(K_convolved) + + return K, K_error + else: + return K + + + + def TritiumSpectrumBackground(self, E=[], endpoint=18.6e3, background=0, m_nu=0, amplitude=1., prob_b=None, prob_c=None, tilt=0., error=False): + + if len(E)==0: + E = self.bin_centers + + if error: + K, K_error = self.TritiumSpectrum(E, endpoint, m_nu, prob_b, prob_c, tilt, error) + K_norm = np.sum(self.TritiumSpectrum(self._energies, endpoint, m_nu, prob_b, prob_c, tilt, error=False)*(self._energies[1]-self._energies[0])) + else: + + K = self.TritiumSpectrum(E, endpoint, m_nu, prob_b, prob_c, tilt, error) + K_norm = np.sum(self.TritiumSpectrum(self._energies, endpoint, m_nu, prob_b, prob_c, tilt, error=False)*(self._energies[1]-self._energies[0])) + + + + + if isinstance(E, float): + print('E is float, only returning tritium amplitude') + return K*(E[1]-E[0])/K_norm + else: + + b = background/(max(self._energies)-min(self._energies)) + a = amplitude#-b + + B = np.ones(np.shape(E)) + B = B*b*(E[1]-E[0])#/(self._energies[1]-self._energies[0]) + #B= B/np.sum(B)*background + + + K = (K/K_norm*(E[1]-E[0]))*a + #K[E>endpoint-np.abs(m_nu**0.5)+de] = np.zeros(len(K[E>endpoint-np.abs(m_nu**2)+de])) + #K= K/np.sum(K)*a + #K = K+B + + + + if error: + K_error = K_error*(E[1]-E[0])/K_norm*a + #K_error = K_error/np.sum(K)*a + return K+B, K_error + else: return K+B + + + def normalized_TritiumSpectrumBackground(self, E=[], endpoint=18.6e3, background=0, m_nu=0., amplitude=1., prob_b=None, prob_c=None, tilt=0., error=False): + + if error: + t, t_error = self.TritiumSpectrumBackground(E, endpoint, background, m_nu, amplitude, prob_b, prob_c, tilt, error=error) + t_norm = np.sum(t) + return t/t_norm, t_error/t_norm + else: + t = self.TritiumSpectrumBackground(E, endpoint, background, m_nu, amplitude, prob_b, prob_c, tilt, error=error) + t_norm = np.sum(t) + return t/t_norm + + + + ################# SNR - Efficiency functions ##################### + + + def Efficiency(self, E, freq=False, pseudo=False): + """ + get efficiencies for energy or frequency bins + + freq: if true, E is assumed to be frequencies instead of energies + """ + + if not freq: + f = self.Frequency(E) + else: + f = deepcopy(E) + E = self.Energy(f) + + + # if E is float + if isinstance(f, float): + #print('Calculating single efficiency value') + f = np.array([f]) + if not phase_4: + df = 1e6 + else: + df = 1 + if pseudo == False: + print('best efficiency') + efficiency, errors = integrated_efficiency(f, self.snr_efficiency_dict, df) + return efficiency, errors + else: + print('pseudo efficiency') + efficiency, errors = pseudo_integrated_efficiency(f, self.snr_efficiency_dict, df, alpha=self.error_scaling) + return efficiency, errors + + + # if E is list or array + else: + + # if efficiency has already been calculated for this array, skip recalculation to save time + if len(f) == len(self._bin_efficiency): + efficiency, errors = self._bin_efficiency, self._bin_efficiency_errors + elif len(f) == len(self._full_efficiency): + efficiency, errors = self._full_efficiency, self._full_efficiency_errors + else: + #logger.info('Calculating efficiency with df = {}...'.format(f[1]-f[0])) + + # toy model efficiency + if self.use_toy_model_efficiency: + efficiency = self.power_eff_interp(f) + errors = self.power_eff_error_interp(f) + + # fss efficiency + else: + efficiency, errors = integrated_efficiency(f, self.snr_efficiency_dict) + + + if self.tilted_efficiency:# and (pseudo or self.fit_efficiency_tilt): + + #efficiency, errors = det_eff.integrated_efficiency(f, self.snr_efficiency_dict) + slope = self.tilt/(1e3) + + logger.info('Tilted efficiencies: {}'.format(self.tilt)) + efficiency *= (1.+slope*(E-17.83e3)) + errors *= (1.+slope*(E-17.83e3)) + + # if we are doing pseudo experiments + if pseudo:# and len(f) < len(self.energies):# and len(f) != len(self._bin_efficiency): + + #logger.info('uncorrelated pseudo efficiencies: {}'.format(self.error_scaling)) + + pseudo_efficiency = np.random.randn(len(f))*np.mean([errors[0], errors[1]], axis=0)*self.error_scaling + #pseudo_efficiency[pseudo_efficiency<0]*=errors[0][pseudo_efficiency<0]*self.error_scaling + #pseudo_efficiency[pseudo_efficiency>=0]*=errors[1][pseudo_efficiency>=0]*self.error_scaling + pseudo_efficiency += efficiency + pseudo_efficiency*=np.sum(efficiency)/np.nansum(pseudo_efficiency) + efficiency = pseudo_efficiency + + if np.min(efficiency) < 0: + index = np.where(E>=np.min(E[efficiency<0])) + efficiency[index] = 0. + + + return efficiency, errors + + + + + +######################################################### +# Functions using model above + + +def GenAndFit(params, counts, fit_config_dict,fit_options, sampled_priors, + i, fixed_data = [], error_scaling=1, tilt = None, fit_tilt = False, event=None): + + scattered = fit_options['scattered'] + distorted = fit_options['distorted'] + fit_nu_mass = fit_options['fit_nu_mass'] + + if i%20 == 0: + logger.info('Sampling: {}'.format(i)) + + + + try: + + T = BinnedTritiumMLFitter("TritiumFitter") + T.InternalConfigure(fit_config_dict) + T.is_scattered = scattered + T.is_distorted = distorted + T.error_scaling = error_scaling + T.integrate_bins = True + + if tilt is not None:# and tilt !=0: + T.tilted_efficiency = True + T.tilt = tilt + + + # generate random data from best fit parameters + if len(fixed_data) == 0: + _, new_data = T.GenerateData(params, counts) + + else: + _, new_data = T.GenerateAsimovData(params) + + + + + + if fit_nu_mass: + # endpoint, background, resolution, signal, nu mass, survival prob + #logger.info('Fitting neutrino mass') + #T.fixes = [False, False, True, False, False, True] + T.fixes[2] = False + + elif fit_tilt: + # endpoint, background, nu_mass, signal, b, c, tilt + T.fixes = [False, False, True, False, True, True, False] + #else: + # # endpoint, background, resolution, signal, nu mass, survival prob + # T.fixes = [False, False, True, False, True, True] + + + + + T.freq_data = new_data + results, errors = T.SampleConvertAndFit(sampled_priors, params) + parameter_samples = T.parameter_samples + + except Exception as e: + print(e) + if event is not None: + event.set() + raise(e) + + else: + return results, errors, parameter_samples + + +def DoOneFit(data, fit_config_dict, fit_options, sampled_parameters={}, error_scaling=0, + tilt=None, fit_tilt = False, data_is_energy=False): + + scattered = fit_options['scattered'] + distorted = fit_options['distorted'] + fit_nu_mass = fit_options['fit_nu_mass'] + + #print('do one fit', tilt, fit_tilt) + T = BinnedTritiumMLFitter("TritiumFitter") + T.InternalConfigure(fit_config_dict) + T.is_scattered = scattered + T.is_distorted = distorted + T.error_scaling = error_scaling + T.integrate_bins = True + logger.info('Energy stepsize: {}'.format(T.denergy)) + + if tilt is not None:# and tilt != 0: + T.tilted_efficiency = True + T.tilt = tilt + + if fit_nu_mass: + logger.info('Fitting neutrino mass') + #T.fixes = [False, False, True, False, False, True] + T.fixes[2] = False + elif fit_tilt: + logger.info('Going to fit efficiency tilt') + T.fixes = [False, False, True, False, True, True, False] + #else: + # T.fixes = [False, False, True, False, True, True] + + if data_is_energy: + data = T.Frequency(data) + T.freq_data = data + results, errors = T.SampleConvertAndFit(sampled_parameters) + total_counts = results[1]+results[3] + + + return results, errors, total_counts + +def GetPDF(fit_config_dict, fit_options, params): + scattered = fit_options['scattered'] + distorted = fit_options['distorted'] + fit_efficiency_tilt = False + + T = BinnedTritiumMLFitter("TritiumFitter") + T.InternalConfigure(fit_config_dict) + T.is_scatterd=scattered + T.is_distorted=distorted + + pdf = T.TritiumSpectrumBackground(T.energies, *params) + _, asimov_binned_data = T.GenerateAsimovData(params) + binned_fit = T.TritiumSpectrumBackground(T.bin_centers, *params, error=True) + + return T.energies, pdf, T.bin_centers, binned_fit, asimov_binned_data diff --git a/mermithid/processors/TritiumSpectrum/__init__.py b/mermithid/processors/TritiumSpectrum/__init__.py index cb5da878..1ffc9520 100644 --- a/mermithid/processors/TritiumSpectrum/__init__.py +++ b/mermithid/processors/TritiumSpectrum/__init__.py @@ -9,4 +9,5 @@ from .DistortedTritiumSpectrumLikelihoodSampler import * from .TritiumSpectrumProcessor import * from .FakeDataGenerator import FakeDataGenerator +from .BinnedTritiumMLFitter import * From 481ec912a9cd38da6712d7764f75b2296339bd5c Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Wed, 4 Aug 2021 15:49:59 -0400 Subject: [PATCH 109/199] Added comments re new input parameters in generator --- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 83b13e5a..5e4876ba 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -70,6 +70,9 @@ def InternalConfigure(self, params): - scattering_sigma [eV]: lineshape parameter - 0-th peak gaussian broadening standard deviation - min_energy [eV]: minimum of lineshape energy window for convolution with beta spectrum. Same magnitude is used for the max energy of the window. - scale_factor: width scaling for a simulated instrumental resolution + - ins_res_width_bounds: Bounds (eV) of regions within which the resolution width is approximately constant (and therefore can be parameterized by a single scale_factor). ins_res_width_bounds does not include the outermost bounds - i.e., the Kmin and Kmax. If ins_res_width_bounds is None, then a common scale factor is used for the whole ROI. + - ins_res_width_factors: Factors that are multiplied by scale_factor in each of the regions defined by ins_res_width_bounds. Note that len(ins_res_width_factors) == len(ins_res_width_bounds) + 1. + - efficiency_path: path to efficiency vs. frequency (and uncertainties) - simplified_scattering_path: path to simplified lineshape parameters – path_to_osc_strengths_files: path to oscillator strength files containing energy loss distributions for the gases in self.gases From dcb10c8c858869042cd1e9ad3fb2132ca61c6d25 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 4 Aug 2021 15:50:48 -0700 Subject: [PATCH 110/199] debugging --- .../Fitters/MCUncertaintyPropagation.py | 8 +- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 266 +++++++++++------- 2 files changed, 171 insertions(+), 103 deletions(-) diff --git a/mermithid/processors/Fitters/MCUncertaintyPropagation.py b/mermithid/processors/Fitters/MCUncertaintyPropagation.py index 1a882dd2..55c9c378 100644 --- a/mermithid/processors/Fitters/MCUncertaintyPropagation.py +++ b/mermithid/processors/Fitters/MCUncertaintyPropagation.py @@ -86,6 +86,7 @@ def ParameterSampling(self): return_dict_keys = ['stat', 'sys', 'combined'] parameter_sampling = [{}, self.sample_parameters, self.sample_parameters] + for k_i, k in enumerate(return_dict_keys): if self.stat_sys_combined[k_i]: @@ -133,10 +134,11 @@ def ParameterSampling(self): parameter_samples_transpose = {} - for p_key in parameter_samples[0].keys(): + for p_key in self.sample_parameters.keys(): parameter_samples_transpose[p_key] = [] - for p_i in parameter_samples: - parameter_samples_transpose[p_key].append(p_i[p_key]) + if len(parameter_samples) > 0 and p_key in parameter_samples[0].keys(): + for p_i in parameter_samples: + parameter_samples_transpose[p_key].append(p_i[p_key]) self.results[k] = {'offsets': [list(o) for o in offsets], diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 659192f8..00f34b40 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -66,14 +66,26 @@ def InternalConfigure(self, config_dict): self.integrate_bins = reader.read_param(config_dict, 'integrate_bins', True) # integrate spectrum over bin widths self.fit_efficiency_tilt = reader.read_param(config_dict, 'fit_efficiency_tilt', False) # efficiency slope is free parameter + self.use_asimov = False + self.fix_nu_mass = not self.fit_nu_mass + self.fix_endpoint = not reader.read_param(config_dict, 'fit_endpoint', True) + self.fix_background = not reader.read_param(config_dict, 'fit_background', True) + self.fix_amplitude = not reader.read_param(config_dict, 'fit_amplitude', True) + self.fix_tilt = not reader.read_param(config_dict, 'fit_tilt', False) + self.fix_scatter_ratio_b = not reader.read_param(config_dict, 'fit_scatter_peak_ratio_b', False) + self.fix_scatter_ratio_c = not reader.read_param(config_dict, 'fit_scatter_peak_ratio_c', False) + self.print_level = 0 + # save plots in - self.savepath = reader.read_param(config_dict, 'plot_path', '.') + self.savepath = reader.read_param(config_dict, 'savepath', '.') # model parameters and uncertainties self.B_mean = reader.read_param(config_dict, 'B_mean', 1) self.B_width = reader.read_param(config_dict, 'B_width', 1e-6) + self.endpoint_mean = reader.read_param(config_dict,'endpoint_mean', 18.6e3) + self.endpoint_width = reader.read_param(config_dict, 'endpoint_width', 1) # efficiency self.efficiency_file_path = reader.read_param(config_dict, 'efficiency_file_path', '') @@ -87,6 +99,7 @@ def InternalConfigure(self, config_dict): # detector response self.NScatters = reader.read_param(config_dict, 'NScatters', 20) self.resolution_model = reader.read_param(config_dict, 'resolution_model', 'gaussian') + self.lineshape_model = reader.read_param(config_dict, 'lineshape_model', 'simplified') self.simplified_lineshape_path = reader.read_param(config_dict, 'simplified_lineshape_path', "required") self.helium_lineshape_path = reader.read_param(config_dict, 'helium_lineshape_path', "optional") self.hydrogen_proportion = reader.read_param(config_dict, 'hydrogen_proportion', 1) @@ -97,42 +110,37 @@ def InternalConfigure(self, config_dict): self.scatter_peak_ratio_c_mean = reader.read_param(config_dict, 'scatter_peak_ratio_c_mean', 0.7) self.scatter_peak_ratio_c_width = reader.read_param(config_dict, 'scatter_peak_ratio_c_width', 0.1) - self.survival_mean = reader.read_param(config_dict, 'survival_mean', 0.5) - self.survival_error = reader.read_param(config_dict, 'survival_width', 0.1) + self.scatter_peak_ratio_mean = reader.read_param(config_dict, 'scatter_peak_ratio_mean', 0.5) + self.scatter_peak_ratio_width = reader.read_param(config_dict, 'scatter_peak_ratio_width', 0.1) - self.res_mean = reader.read_param(config_dict, 'gaussia_resolution_mean', 15.0) + self.res_mean = reader.read_param(config_dict, 'gaussian_resolution_mean', 15.0) self.res_width = reader.read_param(config_dict, 'gaussian_resolution_width', 1.0) - self.two_gaussian_mu1 = reader.read_param(config_dict, 'two_gaussian_mu1', 0) - self.two_gaussian_mu2 = reader.read_param(config_dict, 'two_gaussian_mu2', 0) - self.two_gaussian_sig1_mean = reader.read_param(config_dict, 'two_gaussian_sig1_mean', 0) - self.two_gaussian_sig2_mean = reader.read_param(config_dict, 'two_gaussian_sig2_mean', 0) - self.two_gaussian_sig1_width = reader.read_param(config_dict, 'two_gaussian_sig1_width', 15) - self.two_gaussian_sig2_width = reader.read_param(config_dict, 'two_gaussian_sig2_width', 0) - self.two_gaussian_wide_fraction = reader.read_param(config_dict, 'wide_gaussian_weigth', 1.) + self.two_gaussian_mu_1 = reader.read_param(config_dict, 'two_gaussian_mu1', 0) + self.two_gaussian_mu_2 = reader.read_param(config_dict, 'two_gaussian_mu2', 0) + self.two_gaussian_sig_1_mean = reader.read_param(config_dict, 'two_gaussian_sig1_mean', 15) + self.two_gaussian_sig_2_mean = reader.read_param(config_dict, 'two_gaussian_sig2_mean', 5) + self.two_gaussian_sig_1_width = reader.read_param(config_dict, 'two_gaussian_sig1_width', 1) + self.two_gaussian_sig_2_width = reader.read_param(config_dict, 'two_gaussian_sig2_width', 1) + self.two_gaussian_wide_fraction = reader.read_param(config_dict, 'two_gaussian_wide_fraction', 1.) ######################### # fit configuration ######################## self.counts_guess = reader.read_param(config_dict, 'counts_guess', 5000) - self.fix_counts = False self.mass_guess = reader.read_param(config_dict, 'nu_mass_guess', 0.0) + self.constrained_parameters = reader.read_param(config_dict, 'constrained_parameters', []) + if len(self.constrained_parameters) > 0: + logger.warning('Some parameters are constrained') + self.constrained_means = reader.read_param(config_dict, 'constrained_means', []) + self.constrained_widths = reader.read_param(config_dict, 'constrained_widths', []) # frequency range self.min_frequency = reader.read_param(config_dict, 'min_frequency', "required") self.max_frequency = reader.read_param(config_dict, 'max_frequency', None) - # initial values - self.B = self.B_mean - self.res = self.res_mean - self.scatter_ratio = self.survival_mean - self.endpoint=config_dict['true_endpoint'] - - # some defaults - self.use_asimov = False - ######################################### @@ -184,7 +192,6 @@ def InternalConfigure(self, config_dict): self.helium_lineshape_p = np.loadtxt(self.helium_lineshape_path, unpack=True) # which lineshape should be used? - self.lineshape_model = reader.read_param(config_dict, 'which_lineshape', 'two_gaussian') if self.lineshape_model == 'simplified': self.multi_gas_lineshape = self.simplified_multi_gas_lineshape elif self.lineshape_model == 'accurate_simplified': @@ -232,10 +239,29 @@ def InternalConfigure(self, config_dict): self.use_fixed_scatter_peak_ratio = False else: - logger.error("Configuration of scatter_peak_ratio not known. Options are 'constant' and 'modifed_exponential") + logger.error("Configuration of scatter_peak_ratio not known. Options are 'constant' and 'modified_exponential") raise ValueError("Configuration of scatter_peak_ratio not known. Options are 'constant' and 'efficiency_model") + + ######################################### + # initial values + ######################################### + + self.B = self.B_mean + self.res = self.res_mean + self.endpoint=reader.read_param(config_dict, 'true_endpoint', 18.573e3) + self.two_gaussian_sig_1 = self.two_gaussian_sig_1_mean + self.two_gaussian_sig_2 = self.two_gaussian_sig_2_mean + + if self.use_fixed_scatter_peak_ratio: + self.scatter_peak_ratio_b = self.scatter_peak_ratio_mean + self.scatter_peak_ratio_c = 1 + else: + self.scatter_peak_ratio_b = self.scatter_peak_ratio_b_mean + self.scatter_peak_ratio_c = self.scatter_peak_ratio_c_mean + + ######################################### # energies and bins ######################################### @@ -285,7 +311,14 @@ def InternalConfigure(self, config_dict): # overwrite model self.model = self.TritiumSpectrumBackground + # now configure fit + self.ConfigureFit() + + + return True + + def ConfigureFit(self): # configure fit energy_limits = [max([self.endpoint-300, np.min(self.energies)+3.5]), min([self.endpoint+300, np.max(self.energies)-3.5])] neutrino_limits = [-(np.max(self.energies) - energy_limits[1]-1)**2, (energy_limits[0]-np.min(self.energies)-1)**2] @@ -294,8 +327,8 @@ def InternalConfigure(self, config_dict): #logger.warning('Neutrino mass fitted: {}'.format(self.fit_nu_mass)) if not self.fit_efficiency_tilt: self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', 'scatter_peak_ratio_b', 'scatter_peak_ratio_c'] - self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_c_mean] - self.fixes = [False, False, not self.fit_nu_mass, self.fix_counts, True, True] + self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c] + self.fixes = [self.fix_endpoint, self.fix_background, self.fix_nu_mass, self.fix_amplitude, self.fix_scatter_ratio_b, self.fix_scatter_ratio_c] self.limits = [energy_limits, [1e-10, None], neutrino_limits, @@ -312,9 +345,9 @@ def InternalConfigure(self, config_dict): logger.warning('Efficiency tilt will be fitted') self.tilted_efficiency = True self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', 'Efficiency tilt'] - self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_c_mean, self.tilt] + self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, self.tilt] self.parameter_errors = [max([0.1, 0.1*p]) for p in self.initial_values] - self.fixes = [False, False, not self.fit_nu_mas, self.fix_counts, True, True, False] + self.fixes = [self.fix_endpoint, self.fix_background, self.fix_nu_mass, self.fix_amplitude, self.fix_scatter_ratio_b, self.fix_scatter_ratio_c, self.fix_tilt] self.limits = [energy_limits, [1e-10, None], neutrino_limits, @@ -324,12 +357,10 @@ def InternalConfigure(self, config_dict): [-0.5, 0.5]] - self.constrained_parameters = reader.read_param(config_dict, 'constrained_parameters', []) - if len(self.constrained_parameters) > 0: - logger.warning('Some parameters are constrained') - self.constrained_means = reader.read_param(config_dict, 'constrained_means', []) - self.constrained_widths = reader.read_param(config_dict, 'constrained_widths', []) - self.print_level = 0 + if self.plot_lineshape: + logger.info('Parameters: {}'.format(self.parameter_names)) + logger.info('Fixed: {}'.format(self.fixes)) + logger.info('Initial values: {}'.format(self.initial_values)) return True @@ -498,26 +529,52 @@ def ConvertAndHistogram(self, weights=None): def SamplePriors(self, sampled_parameters): self.parameter_samples = {} sample_values = [] - if 'res' in sampled_parameters.keys(): + if 'res' in sampled_parameters.keys() and sampled_parameters['res']: + self.res = self.Gaussian_sample(self.res_mean, self.res_width) + self.parameter_samples['res'] = self.res + sample_values.append(self.res) + if 'two_gaussian_std_1' in sampled_parameters.keys() and sampled_parameters['two_gaussian_std_1']: + self.two_gaussian_sig_1 = self.Gaussian_sample(self.two_gaussian_sig_1_mean, self.two_gaussian_sig_1_width) + self.parameter_samples['two_gaussian_std_1'] = self.two_gaussian_sig_1 + sample_values.append(self.two_gaussian_sig_1) + if 'two_gaussian_std_2' in sampled_parameters.keys() and sampled_parameters['two_gaussian_std_2']: + self.two_gaussian_sig_2 = self.Gaussian_sample(self.two_gaussian_sig_2_mean, self.two_gaussian_sig_2_width) + self.parameter_samples['two_gaussian_std_2'] = self.two_gaussian_sig_2 + sample_values.append(self.two_gaussian_sig_2) + if 'res' in sampled_parameters.keys() and sampled_parameters['res']: self.res = self.Gaussian_sample(self.res_mean, self.res_width) self.parameter_samples['res'] = self.res sample_values.append(self.res) - if 'scatter_peak_ratio_b' in sampled_parameters.keys(): + if 'scatter_peak_ratio' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio']: + self.scatter_peak_ratio_b = self.Beta_sample(self.scatter_peak_ratio_mean, self.scatter_peak_ratio_width) + self.scatter_peak_ratio_c = 1 + self.fix_scatter_ratio_b = True + self.fix_scatter_ratio_c = True + self.parameter_samples['scatter_peak_ratio'] = self.scatter_peak_ratio_b + sample_values.append(self.scatter_peak_ratio_b) + if 'scatter_peak_ratio_b' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_b']: self.scatter_peak_ratio_b = self.Beta_sample(self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_b_width) + self.fix_scatter_ratio_b = True self.parameter_samples['scatter_peak_ratio_b'] = self.scatter_peak_ratio_b sample_values.append(self.scatter_peak_ratio_b) - if 'scatter_peak_ratio_c' in sampled_parameters.keys(): + if 'scatter_peak_ratio_c' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_c']: self.scatter_peak_ratio_c = self.Beta_sample(self.scatter_peak_ratio_c_mean, self.scatter_peak_ratio_c_width) self.parameter_samples['scatter_peak_ratio_c'] = self.scatter_peak_ratio_c sample_values.append(self.scatter_peak_ratio_c) - if 'B' in sampled_parameters.keys(): + self.fix_scatter_ratio_c = True + if 'B' in sampled_parameters.keys() and sampled_parameters['B']: self.B = self.Gaussian_sample(self.B_mean, self.B_width) self.parameter_samples['B'] = self.B sample_values.append(self.B) + if 'endpoint' in sampled_parameters.keys() and sampled_parameters['endpoint']: + self.endpoint = self.Gaussian_sample(self.endpoint_mean, self.endpoint_width) + self.parameter_samples['endpoint'] = self.endpoint + self.fix_endpoint = True + sample_values.append(self.endpoint) logger.info('Samples are: {}'.format(sample_values)) # set new values in model - self.initial_values[4:6] = [self.scatter_peak_ratio_b, self.scatter_peak_ratio_c] + self.ConfigureFit() return self.parameter_samples @@ -529,9 +586,10 @@ def SampleConvertAndFit(self, sampled_parameters={}, params= []): #self._bin_efficiency, self._bin_efficiency_error = self.Efficiency(self.bin_centers, pseudo=True) self.hist = self.TritiumSpectrumBackground(self.bin_centers, *params) #self.error_scaling = temp + # if random_priors contains 3 items (all boolean) get new sample froom priors if len(sampled_parameters.keys()) > 0: - logger.info('Sampling: {}'.format(sampled_parameters.keys())) + logger.info('Sampling: {}'.format(sampled_parameters)) self.SamplePriors(sampled_parameters) # re-calculate bin efficiencies, if self.pseudo_eff=True efficiency will be ranomized @@ -637,7 +695,7 @@ def which_model(self, *pars): else: return self.effective_TritiumSpectrumShape(*pars) - def scatter_peak_ratio(self, prob_b, prob_c, j): + def mode_exp_scatter_peak_ratio(self, prob_b, prob_c, j): ''' ratio of successive peaks taking reconstruction efficiency into account ''' @@ -651,19 +709,19 @@ def simplified_ls(self, K, Kcenter, FWHM, prob_b, prob_c=1): p0, p1, p2, p3 = self.lineshape_p[1], self.lineshape_p[3], self.lineshape_p[5], self.lineshape_p[7] sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) - shape = gaussian(K, [sig0, Kcenter]) + shape = np.zeros(len(K))#gaussian(K, [sig0, Kcenter]) norm = 1. - for i in range(1, self.NScatters): + for i in range(self.NScatters): sig = p0[i]+p1[i]*FWHM mu = -(p2[i]+p3[i]*np.log(FWHM-30)) if self.use_fixed_scatter_peak_ratio: probi = prob_b**(i+1) else: - probi = self.scatter_peak_ratio(prob_b, prob_c, i) + probi = self.mode_exp_scatter_peak_ratio(prob_b, prob_c, i+1) - shape += probi*gaussian(K, [sig, mu+Kcenter]) + shape += probi*self.gauss_resolution_f(K, 1, sig, mu+Kcenter) norm += probi return shape, norm @@ -690,7 +748,7 @@ def more_accurate_simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) - shape0 = gaussian(K, [sig0, Kcenter]) + shape0 = self.gauss_resolution_f(K, 1, sig0, Kcenter) shape0 *= 1/np.sum(shape0) norm = 1. norm_h = 1. @@ -701,18 +759,18 @@ def more_accurate_simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, #plt.figure(figsize=(10,10)) - for i in range(1, self.NScatters): + for i in range(self.NScatters): # hydrogen scattering sig = p0[i]+p1[i]*FWHM mu = -(p2[i]+p3[i]*np.log(FWHM-30)) if self.use_fixed_scatter_peak_ratio: - probi = prob_b**(i) + probi = prob_b**(i+1) else: - probi = self.scatter_peak_ratio(prob_b, prob_c, i) + probi = self.mode_exp_scatter_peak_ratio(prob_b, prob_c, i+1) - h_scatter_i = gaussian(K, [sig, mu+Kcenter]) + h_scatter_i = self.gauss_resolution_f(K, 1, sig, mu+Kcenter) hydrogen_scattering += probi*h_scatter_i/np.sum(h_scatter_i) norm += probi #plt.plot(K, h_scatter_i, color='blue', label='hydrogen') @@ -754,7 +812,7 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) - shape = gaussian(K, [sig0, Kcenter]) + shape = self.gauss_resolution_f(K, 1, sig0, Kcenter) norm = 1. hydrogen_scattering = np.zeros(len(K)) @@ -762,18 +820,18 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): #plt.figure(figsize=(10,10)) - for i in range(i, self.NScatters): + for i in range(self.NScatters): # hydrogen scattering sig = p0[i]+p1[i]*FWHM mu = -(p2[i]+p3[i]*np.log(FWHM-30)) if self.use_fixed_scatter_peak_ratio: - probi = prob_b**(i) + probi = prob_b**(i+1) else: - probi = self.scatter_peak_ratio(prob_b, prob_c, i) + probi = self.mode_exp_scatter_peak_ratio(prob_b, prob_c, i+1) - h_scatter_i = probi*gaussian(K, [sig, mu+Kcenter]) + h_scatter_i = probi*self.gauss_resolution_f(K, 1, sig, mu+Kcenter) hydrogen_scattering += h_scatter_i norm += probi #plt.plot(K, h_scatter_i, color='blue', label='hydrogen') @@ -848,8 +906,7 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No if self.resolution_model != 'two_gaussian': lineshape = self.gauss_resolution_f(e_lineshape, 1, self.res, 0) else: - lineshape = self.wide_fraction * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_1, self.two_gaussian_mu1) + (1 - self.wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_2, self.two_gaussian_mu2) - + lineshape = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_1, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_2, self.two_gaussian_mu_2) # spectrum shape spec = self.which_model(e_spec, endpoint, m_nu) @@ -902,10 +959,14 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No # get lineshape - if not self.include_helium_scattering: + if not self.use_helium_scattering: tail, norm = self.simplified_ls(e_lineshape, 0, FWHM, prob_b, prob_c) + if self.plot_lineshape: + logger.info('Using simplified lineshape model') else: tail, norm = self.multi_gas_lineshape(e_lineshape, 0, FWHM, prob_b, prob_c) + if self.plot_lineshape: + logger.info('Using two gas simplified lineshape model') lineshape += tail lineshape = lineshape/norm @@ -917,29 +978,46 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No K_convolved = np.interp(self.energies, e_spec, K_convolved) #K_convolved = K_convolved[e_spec>=np.min(self.energies)] - if self.plot_lineshape: - #ax.plot(e_gauss, full_lineshape, label='Full lineshape', color='grey') - # print(FWHM, ratio) - # ax.set_xlabel('Energy [eV]') - # ax.set_ylabel('Amplitude') - # ax.legend(loc='best') - # plt.xlim(-500, 250) - # plt.tight_layout() - # plt.savefig(os.path.join(self.savepath, 'scattering_model.pdf'), dpi=200, transparent=True) - plt.figure(figsize=(7,5)) - #plt.plot(e_lineshape, self.simplified_ls(e_lineshape, 0, FWHM, ratio), color='red', label='FDG') - plt.plot(e_lineshape, lineshape/np.max(lineshape), label = 'Full lineshape', color='Darkblue') + #logger.info('Plotting lineshape now: {}'.format(self.plot_lineshape)) + if self.plot_lineshape: + logger.info('Plotting confirmed') + #ax.plot(e_gauss, full_lineshape, label='Full lineshape', color='grey') + # print(FWHM, ratio) + # ax.set_xlabel('Energy [eV]') + # ax.set_ylabel('Amplitude') + # ax.legend(loc='best') + # plt.xlim(-500, 250) + # plt.tight_layout() + # plt.savefig(os.path.join(self.savepath, 'scattering_model.pdf'), dpi=200, transparent=True) + + plt.figure(figsize=(7,5)) + #plt.plot(e_lineshape, self.simplified_ls(e_lineshape, 0, FWHM, ratio), color='red', label='FDG') + if self.resolution_model != 'two_gaussian': + resolution = self.gauss_resolution_f(e_lineshape, 1, self.res, 0) + if self.plot_lineshape: + logger.info('Using Gaussian resolution model') + else: + resolution = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_1, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_2, self.two_gaussian_mu_2) + if self.plot_lineshape: + logger.info('Using two Gaussian resolution model') - simple_ls = self.simplified_ls(e_lineshape, 0, FWHM, ratio) - plt.plot(e_lineshape, simple_ls/np.max(simple_ls), label='Hydrogen only lineshape', color='red') - plt.xlabel('Energy [eV]') - plt.ylabel('Amplitude') - #plt.grid() - plt.xlim(-500, 250) - plt.legend(loc='best') - plt.tight_layout() - plt.savefig(os.path.join(self.savepath, 'lineshape.pdf'), dpi=200, transparent=True) + + plt.plot(e_lineshape, resolution/np.max(resolution), label = 'Resolution', color='orange') + plt.plot(e_lineshape, lineshape/np.max(lineshape), label = 'Full lineshape', color='Darkblue') + + FWHM = 2.*np.sqrt(2.*np.log(2.))*self.res + print(prob_b, prob_c) + simple_ls, simple_norm = self.simplified_ls(e_lineshape, 0, FWHM, prob_b, prob_c) + simple_ls = (self.gauss_resolution_f(e_lineshape, 1, self.res, 0)+simple_ls)/simple_norm + plt.plot(e_lineshape, simple_ls/np.nanmax(simple_ls), label='Hydrogen only lineshape', color='red') + plt.xlabel('Energy [eV]') + plt.ylabel('Amplitude') + #plt.grid() + plt.xlim(-500, 250) + plt.legend(loc='best') + plt.tight_layout() + plt.savefig(os.path.join(self.savepath, 'lineshape.pdf'), dpi=200, transparent=True) @@ -1192,6 +1270,7 @@ def GenAndFit(params, counts, fit_config_dict,fit_options, sampled_priors, T.tilt = tilt + # generate random data from best fit parameters if len(fixed_data) == 0: _, new_data = T.GenerateData(params, counts) @@ -1200,24 +1279,11 @@ def GenAndFit(params, counts, fit_config_dict,fit_options, sampled_priors, _, new_data = T.GenerateAsimovData(params) - - - if fit_nu_mass: - # endpoint, background, resolution, signal, nu mass, survival prob - #logger.info('Fitting neutrino mass') - #T.fixes = [False, False, True, False, False, True] - T.fixes[2] = False + T.fix_nu_mass = False elif fit_tilt: - # endpoint, background, nu_mass, signal, b, c, tilt - T.fixes = [False, False, True, False, True, True, False] - #else: - # # endpoint, background, resolution, signal, nu mass, survival prob - # T.fixes = [False, False, True, False, True, True] - - - + T.fix_tilt = False T.freq_data = new_data results, errors = T.SampleConvertAndFit(sampled_priors, params) @@ -1255,13 +1321,10 @@ def DoOneFit(data, fit_config_dict, fit_options, sampled_parameters={}, error_sc if fit_nu_mass: logger.info('Fitting neutrino mass') - #T.fixes = [False, False, True, False, False, True] - T.fixes[2] = False + T.fix_nu_mass = False elif fit_tilt: logger.info('Going to fit efficiency tilt') - T.fixes = [False, False, True, False, True, True, False] - #else: - # T.fixes = [False, False, True, False, True, True] + T.fix_tilt = False if data_is_energy: data = T.Frequency(data) @@ -1272,13 +1335,16 @@ def DoOneFit(data, fit_config_dict, fit_options, sampled_parameters={}, error_sc return results, errors, total_counts -def GetPDF(fit_config_dict, fit_options, params): +def GetPDF(fit_config_dict, fit_options, params, plot=False): + logger.info('Plotting lineshape: {}'.format(plot)) + logger.info('PDF for params: {}'.format(params)) + logger.info(fit_options) scattered = fit_options['scattered'] distorted = fit_options['distorted'] - fit_efficiency_tilt = False T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) + T.plot_lineshape = plot T.is_scatterd=scattered T.is_distorted=distorted From 9052050781fc6d6fa81f2108ea1f38ab44549fbb Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 5 Aug 2021 13:17:21 -0700 Subject: [PATCH 111/199] two small fixes --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 00f34b40..e9a8dd5b 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -441,6 +441,11 @@ def bins(self, some_bins): # energy bins self._bins = some_bins + if len(self._energies) > self.N_energy_bins: + self._energies = self._energies[:-1] + if len(self._bins) > self.N_bins: + self._bins = self._bins[:-1] + self.bin_centers = self._bins[0:-1] +0.5*(self._bins[1]-self._bins[0]) # frequency bins @@ -457,7 +462,7 @@ def ReSetBins(self): #self.bins = np.arange(np.min(self.energies), np.max(self.energies), self.dbins) self.energies = np.arange(self.Energy(self.max_frequency), self.Energy(self.max_frequency)+(self.N_energy_bins)*self.denergy, self.denergy) - self.bins = np.arange(np.min(self.energies), self.Energy(self.max_frequency)+(self.N_bins)*self.dbins, self.dbins) + self.bins = np.arange(np.min(self.energies), np.min(self.energies)+(self.N_bins)*self.dbins, self.dbins) self._bin_efficiency, self._bin_efficiency_error = [], [] if len(self._energies) > self.N_energy_bins: @@ -541,10 +546,6 @@ def SamplePriors(self, sampled_parameters): self.two_gaussian_sig_2 = self.Gaussian_sample(self.two_gaussian_sig_2_mean, self.two_gaussian_sig_2_width) self.parameter_samples['two_gaussian_std_2'] = self.two_gaussian_sig_2 sample_values.append(self.two_gaussian_sig_2) - if 'res' in sampled_parameters.keys() and sampled_parameters['res']: - self.res = self.Gaussian_sample(self.res_mean, self.res_width) - self.parameter_samples['res'] = self.res - sample_values.append(self.res) if 'scatter_peak_ratio' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio']: self.scatter_peak_ratio_b = self.Beta_sample(self.scatter_peak_ratio_mean, self.scatter_peak_ratio_width) self.scatter_peak_ratio_c = 1 From 0c25da8f7a021dbe48839cffc9db597db4edfca5 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Fri, 6 Aug 2021 15:14:22 -0700 Subject: [PATCH 112/199] fixing gaussian resolution with detailed lineshape combo --- mermithid/misc/FakeTritiumDataFunctions.py | 29 +++++++++---------- .../TritiumSpectrum/FakeDataGenerator.py | 15 ++++++---- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 255f8712..e96bdbdd 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -305,31 +305,30 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - if resolution_function == 'simulated_resolution' or 'simulated': + if resolution_function == 'simulated_resolution' or resolution_function == 'simulated': lineshape_rates = [] scale_factors = [ls_params[0]*f for f in ins_res_width_factors] for scale in scale_factors: lineshape_rates.append(np.flipud(complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac'))) - elif resolution_function == 'gaussian_resolution' or 'gaussian': + elif resolution_function == 'gaussian_resolution' or resolution_function == 'gaussian': lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') else: logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) #Convolving - if lineshape=='detailed_scattering' or lineshape=='detailed': - if resolution_function == 'simulated_resolution' or 'simulated': - convolved_list = [] - for j in range(len(lineshape_rates)): - beta_rates = spectral_rate(K_segments[j], Q, mnu, final_state_array) - convolved_list.append(convolve(beta_rates, lineshape_rates[j], mode='same')) - convolved = np.concatenate(convolved_list, axis=None) + if (lineshape=='detailed_scattering' or lineshape=='detailed') and (resolution_function == 'simulated_resolution' or resolution_function == 'simulated'): + convolved_list = [] + for j in range(len(lineshape_rates)): + beta_rates = spectral_rate(K_segments[j], Q, mnu, final_state_array) + convolved_list.append(convolve(beta_rates, lineshape_rates[j], mode='same')) + convolved = np.concatenate(convolved_list, axis=None) else: lineshape_rates = np.flipud(lineshape_rates) beta_rates = spectral_rate(K, Q, mnu, final_state_array) convolved = convolve(beta_rates, lineshape_rates, mode='same') - - + + below_Kmin = np.where(K < Kmin) np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) return convolved @@ -353,14 +352,14 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak elif lineshape=='simplified_scattering' or lineshape=='simplified': lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': - if resolution_function == 'simulated_resolution' or 'simulated': + if resolution_function == 'simulated_resolution' or resolution_function == 'simulated': lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') - elif resolution_function == 'gaussian_resolution' or 'gaussian': + elif resolution_function == 'gaussian_resolution' or resolution_function == 'gaussian': lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') else: logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) lineshape_rates = np.flipud(lineshape_rates) - + bkgd_rates = np.full(len(K), bkgd_rate()) if len(K) < len(K_lineshape): raise Exception("lineshape array is longer than Koptions") @@ -369,7 +368,7 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak convolved = convolve(bkgd_rates, lineshape_rates, mode='same') below_Kmin = np.where(K < Kmin) np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) - + return convolved diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 5e4876ba..364803ec 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -72,7 +72,7 @@ def InternalConfigure(self, params): - scale_factor: width scaling for a simulated instrumental resolution - ins_res_width_bounds: Bounds (eV) of regions within which the resolution width is approximately constant (and therefore can be parameterized by a single scale_factor). ins_res_width_bounds does not include the outermost bounds - i.e., the Kmin and Kmax. If ins_res_width_bounds is None, then a common scale factor is used for the whole ROI. - ins_res_width_factors: Factors that are multiplied by scale_factor in each of the regions defined by ins_res_width_bounds. Note that len(ins_res_width_factors) == len(ins_res_width_bounds) + 1. - + - efficiency_path: path to efficiency vs. frequency (and uncertainties) - simplified_scattering_path: path to simplified lineshape parameters – path_to_osc_strengths_files: path to oscillator strength files containing energy loss distributions for the gases in self.gases @@ -194,8 +194,11 @@ def InternalConfigure(self, params): # lineshape params - self.ls_params = [self.scale_factor, self.survival_prob] - + if self.resolution_function == 'gaussian_resolution' or self.resolution_function == 'gaussian': + self.ls_params = [self.scattering_sigma*2*math.sqrt(2*math.log(2)), self.survival_prob] + else: + self.ls_params = [self.scale_factor, self.survival_prob] + # Setup and configure lineshape processor complexLineShape_config = { 'gases': self.gases, @@ -219,7 +222,7 @@ def InternalConfigure(self, params): 'gaussian_proportion': self.gaussian_proportion, 'A_array': self.A_array, 'sigma_array': self.sigma_array, - + # This is an important parameter which determines how finely resolved the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to be increased for larger datasets. 'num_points_in_std_array':35846, 'base_shape': 'dirac', @@ -229,7 +232,7 @@ def InternalConfigure(self, params): 'path_to_ins_resolution_data_txt': self.path_to_ins_resolution_data_txt, 'use_combined_four_trap_inst_reso': self.use_combined_four_trap_inst_reso, 'path_to_four_trap_ins_resolution_data_txt': self.path_to_four_trap_ins_resolution_data_txt, - 'shake_spectrum_parameters_json_path': self.shake_spectrum_parameters_json_path + 'shake_spectrum_parameters_json_path': self.shake_spectrum_parameters_json_path } logger.info('Setting up complex lineshape object') self.complexLineShape = MultiGasComplexLineShape("complexLineShape") @@ -349,7 +352,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 else: Kmin, Kmax = ROIbound[0], ROIbound[1] B = B_1kev*(Kmax-Kmin)/1000. - + nstdevs = 7 #Number of standard deviations (of size broadening) below Kmin and above Q-m to generate data, for the gaussian case FWHM_convert = 2*math.sqrt(2*math.log(2)) max_energy = -self.min_energy From d0d7b1138f8dfda9c959d941558c4833f436956b Mon Sep 17 00:00:00 2001 From: cclaessens Date: Fri, 6 Aug 2021 17:17:41 -0700 Subject: [PATCH 113/199] fixes in helium lineshape model --- .../Fitters/MCUncertaintyPropagation.py | 3 +- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 37 +++++++++++++------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/mermithid/processors/Fitters/MCUncertaintyPropagation.py b/mermithid/processors/Fitters/MCUncertaintyPropagation.py index 55c9c378..57e1c518 100644 --- a/mermithid/processors/Fitters/MCUncertaintyPropagation.py +++ b/mermithid/processors/Fitters/MCUncertaintyPropagation.py @@ -145,7 +145,8 @@ def ParameterSampling(self): 'sigmas': [list(s) for s in sigmas], 'results': [list(fr) for fr in fit_results], 'parameter_samples': parameter_samples_transpose, - 'best_fit': list(self.fitted_params) + 'best_fit': list(self.fitted_params), + 'best_fit_errors': list(self.fitted_params_errors) } diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index e9a8dd5b..a031f3d5 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -130,11 +130,13 @@ def InternalConfigure(self, config_dict): ######################## self.counts_guess = reader.read_param(config_dict, 'counts_guess', 5000) self.mass_guess = reader.read_param(config_dict, 'nu_mass_guess', 0.0) + self.constrained_parameter_names = reader.read_param(config_dict, 'constrained_parameter_names', []) self.constrained_parameters = reader.read_param(config_dict, 'constrained_parameters', []) - if len(self.constrained_parameters) > 0: - logger.warning('Some parameters are constrained') self.constrained_means = reader.read_param(config_dict, 'constrained_means', []) self.constrained_widths = reader.read_param(config_dict, 'constrained_widths', []) + if len(self.constrained_parameter_names) > 0: + logger.warning('Some parameters are constrained: {} - {}'.format(self.constrained_parameters, self.constrained_parameter_names)) + #self.print_level = 1 # frequency range self.min_frequency = reader.read_param(config_dict, 'min_frequency', "required") @@ -362,6 +364,12 @@ def ConfigureFit(self): logger.info('Fixed: {}'.format(self.fixes)) logger.info('Initial values: {}'.format(self.initial_values)) + if len(self.constrained_parameters) > 0: + #logger.info('{}\n{}'.format(self.parameter_names, self.constrained_parameter_names)) + #self.constrained_parameters = [self.parameter_names.index(p) for p in self.constrained_parameter_names[0]] + self.print_level=1 + + return True @@ -574,6 +582,7 @@ def SamplePriors(self, sampled_parameters): sample_values.append(self.endpoint) logger.info('Samples are: {}'.format(sample_values)) + #logger.info('Fit parameters: \n{}\nFixed: {}'.format(self.parameter_names, self.fixes)) # set new values in model self.ConfigureFit() @@ -638,7 +647,11 @@ def approximate_spectrum(self, E, Q, m_nu=0): but the ephasespace is approximate (some factors neglected) """ - mnu = np.real(m_nu**0.5) + # mnu is used in heaviside function + if m_nu >=0: + mnu = np.abs(m_nu**0.5) + else: + mnu = 0 if self.use_final_states: if isinstance(E, list) or isinstance(E, np.ndarray): @@ -749,11 +762,12 @@ def more_accurate_simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) - shape0 = self.gauss_resolution_f(K, 1, sig0, Kcenter) - shape0 *= 1/np.sum(shape0) + #shape0 = self.gauss_resolution_f(K, 1, sig0, Kcenter) + #shape0 *= 1/np.sum(shape0) + shape0 = np.zeros(len(K)) norm = 1. norm_h = 1. - norm_h2 = 1. + norm_he = 1. hydrogen_scattering = np.zeros(len(K)) helium_scattering = np.zeros(len(K)) @@ -789,8 +803,8 @@ def more_accurate_simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, #plt.plot(K, (shape0 + helium_scattering)/np.max(shape0 + helium_scattering), color='red', label='helium') # full lineshape - norm_h = np.sum(shape0 + hydrogen_scattering) - norm_he = np.sum(shape0 + helium_scattering) + #norm_h = np.sum(shape0 + hydrogen_scattering) + #norm_he = np.sum(shape0 + helium_scattering) lineshape = (self.hydrogen_proportion*(shape0 + hydrogen_scattering)/norm_h + (1-self.hydrogen_proportion)*(shape0 + helium_scattering)/norm_he) @@ -813,7 +827,8 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) - shape = self.gauss_resolution_f(K, 1, sig0, Kcenter) + #shape = self.gauss_resolution_f(K, 1, sig0, Kcenter) + shape = np.zeros(len(K)) norm = 1. hydrogen_scattering = np.zeros(len(K)) @@ -1008,13 +1023,13 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No plt.plot(e_lineshape, lineshape/np.max(lineshape), label = 'Full lineshape', color='Darkblue') FWHM = 2.*np.sqrt(2.*np.log(2.))*self.res - print(prob_b, prob_c) + print(prob_b, prob_c, FWHM) simple_ls, simple_norm = self.simplified_ls(e_lineshape, 0, FWHM, prob_b, prob_c) simple_ls = (self.gauss_resolution_f(e_lineshape, 1, self.res, 0)+simple_ls)/simple_norm plt.plot(e_lineshape, simple_ls/np.nanmax(simple_ls), label='Hydrogen only lineshape', color='red') plt.xlabel('Energy [eV]') plt.ylabel('Amplitude') - #plt.grid() + plt.grid() plt.xlim(-500, 250) plt.legend(loc='best') plt.tight_layout() From f0959e2336b190ae897025df85d86957da388bc9 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Sat, 7 Aug 2021 11:24:32 -0400 Subject: [PATCH 114/199] Re-introduced trap weight and resolution bin sampling --- mermithid/processors/misc/MultiGasComplexLineShape.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 59b6b449..fa05181d 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -514,7 +514,15 @@ def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_a return normalized_convolved_spectrum def convolve_simulated_resolution_scaled(self, working_spectrum, scale_factor): - x_data, y_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + if self.use_combined_four_trap_inst_reso: + x_data, y_data, y_err_data = self.combine_four_trap_resolution_from_txt(self.trap_weights) + logger.info("Combined four instrumental resolution files") + else: + x_data, y_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + logger.info("Using ONE simulated instrumental resolution file (not combining four)") + if self.sample_ins_resolution_errors: + y_data = np.random.normal(y_data, y_err_data) + logger.info("Sampling instrumental resolution counts per bin") scaled_xdata = x_data*scale_factor f = interpolate.interp1d(x_data*scale_factor, y_data) x_array = self.std_eV_array() @@ -3112,6 +3120,7 @@ def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() elif emitted_peak == 'dirac': current_working_spectrum = self.std_dirac() + current_working_spectrum = self.convolve_simulated_resolution_scaled(current_working_spectrum, scale_factor) zeroth_order_peak = current_working_spectrum current_full_spectrum += zeroth_order_peak From 232928338f94d81b0cd68471a795c0b1c82d6cef Mon Sep 17 00:00:00 2001 From: cclaessens Date: Tue, 10 Aug 2021 10:44:45 +0200 Subject: [PATCH 115/199] sampling b and c from gamma --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index a031f3d5..fde12c03 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -380,10 +380,17 @@ def Gaussian_sample(self, mean, width): return np.random.randn()*width+mean def Beta_sample(self, mean, width): + np.random.seed() a = ((1-mean)/(width**2)-1/mean)*mean**2 b = (1/mean-1)*a return np.random.beta(a, b) + def Gamma_sample(self, mean, width): + np.random.seed() + a = (mean/width)**2 + b = mean/(width**2) + return np.random.gamma(a, 1/b) + ############################ conversion methods ########################### @@ -540,6 +547,8 @@ def ConvertAndHistogram(self, weights=None): def SamplePriors(self, sampled_parameters): + logger.info('Sampling: {}'.format([k for k in sampled_parameters.keys() if sampled_parameters[k]])) + self.parameter_samples = {} sample_values = [] if 'res' in sampled_parameters.keys() and sampled_parameters['res']: @@ -562,12 +571,12 @@ def SamplePriors(self, sampled_parameters): self.parameter_samples['scatter_peak_ratio'] = self.scatter_peak_ratio_b sample_values.append(self.scatter_peak_ratio_b) if 'scatter_peak_ratio_b' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_b']: - self.scatter_peak_ratio_b = self.Beta_sample(self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_b_width) + self.scatter_peak_ratio_b = self.Gamma_sample(self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_b_width) self.fix_scatter_ratio_b = True self.parameter_samples['scatter_peak_ratio_b'] = self.scatter_peak_ratio_b sample_values.append(self.scatter_peak_ratio_b) if 'scatter_peak_ratio_c' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_c']: - self.scatter_peak_ratio_c = self.Beta_sample(self.scatter_peak_ratio_c_mean, self.scatter_peak_ratio_c_width) + self.scatter_peak_ratio_c = self.Gamma_sample(self.scatter_peak_ratio_c_mean, self.scatter_peak_ratio_c_width) self.parameter_samples['scatter_peak_ratio_c'] = self.scatter_peak_ratio_c sample_values.append(self.scatter_peak_ratio_c) self.fix_scatter_ratio_c = True @@ -599,7 +608,6 @@ def SampleConvertAndFit(self, sampled_parameters={}, params= []): # if random_priors contains 3 items (all boolean) get new sample froom priors if len(sampled_parameters.keys()) > 0: - logger.info('Sampling: {}'.format(sampled_parameters)) self.SamplePriors(sampled_parameters) # re-calculate bin efficiencies, if self.pseudo_eff=True efficiency will be ranomized @@ -1022,8 +1030,9 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No plt.plot(e_lineshape, resolution/np.max(resolution), label = 'Resolution', color='orange') plt.plot(e_lineshape, lineshape/np.max(lineshape), label = 'Full lineshape', color='Darkblue') + FWHM = 2.*np.sqrt(2.*np.log(2.))*self.res - print(prob_b, prob_c, FWHM) + logger.info('Plotting lineshape for FWHM {}, probs {} and {} and hydrogen proportion {}.'.format(FWHM, prob_b, prob_c, self.hydrogen_proportion)) simple_ls, simple_norm = self.simplified_ls(e_lineshape, 0, FWHM, prob_b, prob_c) simple_ls = (self.gauss_resolution_f(e_lineshape, 1, self.res, 0)+simple_ls)/simple_norm plt.plot(e_lineshape, simple_ls/np.nanmax(simple_ls), label='Hydrogen only lineshape', color='red') From 0d3350f1d361f9b56a5e20b11595e63019957fdc Mon Sep 17 00:00:00 2001 From: Jula Date: Tue, 10 Aug 2021 21:38:44 -0400 Subject: [PATCH 116/199] fixed molecular final states file and added extended --- mermithid/misc/saenz_mfs.json | 2 +- mermithid/misc/saenz_mfs_ctd.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 mermithid/misc/saenz_mfs_ctd.json diff --git a/mermithid/misc/saenz_mfs.json b/mermithid/misc/saenz_mfs.json index 3d3b9d7d..89524096 100644 --- a/mermithid/misc/saenz_mfs.json +++ b/mermithid/misc/saenz_mfs.json @@ -1 +1 @@ -{"Binding energy": [1.897, 1.844, 1.773, 1.65, 1.546, 1.455, 1.341, 1.232, 1.138, 1.047, 0.96, 0.849, 0.754, 0.647999999999999, 0.538, 0.446, 0.345, 0.24, 0.151999999999999, 0.0629999999999999, -0.0429999999999999, -0.147, -0.247, -0.347, -0.446999999999999, -0.613, -0.865, -1.112, -1.36, -1.61, -1.86, -2.186, -2.68199999999999, -3.23499999999999, -3.75, -16.603, -17.603, -18.799, -19.761, -20.73, -21.701, -22.676, -23.653, -24.632, -25.613, -26.596, -27.581, -28.567, -29.558, -30.593, -31.66, -32.637, -33.595, -34.562, -35.548, -36.566, -37.602, -38.609, -39.601, -40.601, -41.607, -42.614, -43.597, -44.584, -45.586, -46.616, -47.601, -48.565, -49.604, -50.599, -51.594, -52.605, -53.611, -54.629, -55.621, -56.632, -57.621, -58.608, -59.608, -60.604, -61.133, -62.615, -63.607, -64.613, -65.604, -66.595, -67.592, -68.589, -69.5759999999999, -70.5809999999999, -71.589, -72.5459999999999, -73.5489999999999, -74.568, -75.533, -76.6149999999999, -77.5669999999999, -78.607, -79.613, -80.6259999999999, -81.6079999999999, -82.6019999999999, -83.5929999999999, -84.5939999999999, -85.601, -86.601, -87.598, -89.0699999999999, -91.086, -93.0849999999999, -95.0849999999999, -97.084, -99.084, -101.086, -103.087, -105.088, -107.089, -109.089, -111.09, -113.09, -115.09, -117.091, -119.091, -121.091, -123.092, -125.092, -127.092, -129.092, -131.093, -133.093, -135.093, -137.093, -140.065, -144.067, -148.068, -152.069, -156.07, -160.072, -164.073, -168.074, -172.075, -176.076, -180.076, -184.077, -188.078, -192.079, -196.079, -200.08, -204.08, -208.081, -212.082, -216.082, -220.082, -224.083, -228.083, -232.084, -236.084], "Probability": [9.99999999999999e-08, 0.0006900000000000001, 0.00046, 0.00233, 0.00553, 0.00457, 0.02033, 0.01649, 0.03877, 0.038079999999999996, 0.06809, 0.11214, 0.10112, 0.24406, 0.32337000000000005, 0.40864, 0.68745, 0.66279, 0.51412, 0.6556100000000001, 0.54588, 0.37231000000000003, 0.25473, 0.16959, 0.11369, 0.16946999999999998, 0.10094, 0.05732, 0.02806, 0.013160000000000002, 0.00623, 0.0042, 0.0008, 0.00015, 0.0, 0.0, 0.0, 1.1999999999999999e-05, 0.000113, 0.0006560000000000001, 0.002567, 0.007149, 0.014804, 0.023583, 0.029715, 0.030307, 0.025527, 0.018080000000000002, 0.01107, 0.007377000000000001, 0.010637, 0.019095, 0.022178, 0.016434, 0.009037, 0.004989, 0.003978, 0.004124, 0.004152, 0.0039250000000000005, 0.003457, 0.003186, 0.0027010000000000003, 0.0027129999999999997, 0.002481, 0.002412, 0.001907, 0.001938, 0.0017599999999999998, 0.001575, 0.0015409999999999998, 0.001485, 0.001557, 0.001895, 0.002427, 0.003357, 0.004095, 0.004714, 0.005033999999999999, 0.005152, 0.005442000000000001, 0.005859, 0.006617, 0.0070940000000000005, 0.007404, 0.007164, 0.006563, 0.005620000000000001, 0.004691, 0.00368, 0.003049, 0.00221, 0.001928, 0.0017610000000000002, 0.0015300000000000001, 0.001215, 0.0013900000000000002, 0.001216, 0.0014219999999999999, 0.001384, 0.001368, 0.001316, 0.001153, 0.0010760000000000001, 0.000921, 0.0007570000000000001, 0.000696, 0.0006180000000000001, 0.00054, 0.00048300000000000003, 0.00043200000000000004, 0.000388, 0.00035150000000000003, 0.00031800000000000003, 0.000289, 0.000264, 0.00024150000000000002, 0.000222, 0.0002045, 0.000189, 0.00017500000000000003, 0.00016250000000000002, 0.000151, 0.000141, 0.0001315, 0.000123, 0.000115, 0.00010800000000000001, 0.0001015, 9.549999999999999e-05, 8.999999999999999e-05, 8.45e-05, 7.775e-05, 6.95e-05, 6.25e-05, 5.625e-05, 5.1000000000000006e-05, 4.625e-05, 4.225e-05, 3.85e-05, 3.525e-05, 3.25e-05, 3e-05, 2.775e-05, 2.5750000000000002e-05, 2.375e-05, 2.225e-05, 2.075e-05, 1.925e-05, 1.8e-05, 1.7e-05, 1.6000000000000003e-05, 1.5e-05, 1.4e-05, 1.325e-05, 1.1750000000000001e-05, NaN]} \ No newline at end of file +{"Binding energy": [1.897, 1.844, 1.773, 1.65, 1.546, 1.455, 1.341, 1.232, 1.138, 1.047, 0.96, 0.849, 0.754, 0.647999999999999, 0.538, 0.446, 0.345, 0.24, 0.151999999999999, 0.0629999999999999, -0.0429999999999999, -0.147, -0.247, -0.347, -0.446999999999999, -0.613, -0.865, -1.112, -1.36, -1.61, -1.86, -2.186, -2.68199999999999, -3.23499999999999, -3.75, -16.603, -17.603, -18.799, -19.761, -20.73, -21.701, -22.676, -23.653, -24.632, -25.613, -26.596, -27.581, -28.567, -29.558, -30.593, -31.66, -32.637, -33.595, -34.562, -35.548, -36.566, -37.602, -38.609, -39.601, -40.601, -41.607, -42.614, -43.597, -44.584, -45.586, -46.616, -47.601, -48.565, -49.604, -50.599, -51.594, -52.605, -53.611, -54.629, -55.621, -56.632, -57.621, -58.608, -59.608, -60.604, -61.133, -62.615, -63.607, -64.613, -65.604, -66.595, -67.592, -68.589, -69.5759999999999, -70.5809999999999, -71.589, -72.5459999999999, -73.5489999999999, -74.568, -75.533, -76.6149999999999, -77.5669999999999, -78.607, -79.613, -80.6259999999999, -81.6079999999999, -82.6019999999999, -83.5929999999999, -84.5939999999999, -85.601, -86.601, -87.598, -89.0699999999999, -91.086, -93.0849999999999, -95.0849999999999, -97.084, -99.084, -101.086, -103.087, -105.088, -107.089, -109.089, -111.09, -113.09, -115.09, -117.091, -119.091, -121.091, -123.092, -125.092, -127.092, -129.092, -131.093, -133.093, -135.093, -137.093, -140.065, -144.067, -148.068, -152.069, -156.07, -160.072, -164.073, -168.074, -172.075, -176.076, -180.076, -184.077, -188.078, -192.079, -196.079, -200.08, -204.08, -208.081, -212.082, -216.082, -220.082, -224.083, -228.083, -232.084, -236.084], "Probability": [9.99999999999999e-08, 0.0006900000000000001, 0.00046, 0.00233, 0.00553, 0.00457, 0.02033, 0.01649, 0.03877, 0.038079999999999996, 0.06809, 0.11214, 0.10112, 0.24406, 0.32337000000000005, 0.40864, 0.68745, 0.66279, 0.51412, 0.6556100000000001, 0.54588, 0.37231000000000003, 0.25473, 0.16959, 0.11369, 0.16946999999999998, 0.10094, 0.05732, 0.02806, 0.013160000000000002, 0.00623, 0.0042, 0.0008, 0.00015, 0.0, 0.0, 0.0, 1.1999999999999999e-05, 0.000113, 0.0006560000000000001, 0.002567, 0.007149, 0.014804, 0.023583, 0.029715, 0.030307, 0.025527, 0.018080000000000002, 0.01107, 0.007377000000000001, 0.010637, 0.019095, 0.022178, 0.016434, 0.009037, 0.004989, 0.003978, 0.004124, 0.004152, 0.0039250000000000005, 0.003457, 0.003186, 0.0027010000000000003, 0.0027129999999999997, 0.002481, 0.002412, 0.001907, 0.001938, 0.0017599999999999998, 0.001575, 0.0015409999999999998, 0.001485, 0.001557, 0.001895, 0.002427, 0.003357, 0.004095, 0.004714, 0.005033999999999999, 0.005152, 0.005442000000000001, 0.005859, 0.006617, 0.0070940000000000005, 0.007404, 0.007164, 0.006563, 0.005620000000000001, 0.004691, 0.00368, 0.003049, 0.00221, 0.001928, 0.0017610000000000002, 0.0015300000000000001, 0.001215, 0.0013900000000000002, 0.001216, 0.0014219999999999999, 0.001384, 0.001368, 0.001316, 0.001153, 0.0010760000000000001, 0.000921, 0.0007570000000000001, 0.000696, 0.0006180000000000001, 0.00054, 0.00048300000000000003, 0.00043200000000000004, 0.000388, 0.00035150000000000003, 0.00031800000000000003, 0.000289, 0.000264, 0.00024150000000000002, 0.000222, 0.0002045, 0.000189, 0.00017500000000000003, 0.00016250000000000002, 0.000151, 0.000141, 0.0001315, 0.000123, 0.000115, 0.00010800000000000001, 0.0001015, 9.549999999999999e-05, 8.999999999999999e-05, 8.45e-05, 7.775e-05, 6.95e-05, 6.25e-05, 5.625e-05, 5.1000000000000006e-05, 4.625e-05, 4.225e-05, 3.85e-05, 3.525e-05, 3.25e-05, 3e-05, 2.775e-05, 2.5750000000000002e-05, 2.375e-05, 2.225e-05, 2.075e-05, 1.925e-05, 1.8e-05, 1.7e-05, 1.6000000000000003e-05, 1.5e-05, 1.4e-05, 1.325e-05, 1.25e-05, 1.1750000000000001e-05]} diff --git a/mermithid/misc/saenz_mfs_ctd.json b/mermithid/misc/saenz_mfs_ctd.json new file mode 100644 index 00000000..1c78c976 --- /dev/null +++ b/mermithid/misc/saenz_mfs_ctd.json @@ -0,0 +1 @@ +{"Binding energy": [1.897, 1.844, 1.773, 1.65, 1.546, 1.455, 1.341, 1.232, 1.138, 1.047, 0.96, 0.849, 0.754, 0.647999999999999, 0.538, 0.446, 0.345, 0.24, 0.151999999999999, 0.0629999999999999, -0.0429999999999999, -0.147, -0.247, -0.347, -0.446999999999999, -0.613, -0.865, -1.112, -1.36, -1.61, -1.86, -2.186, -2.68199999999999, -3.23499999999999, -3.75, -16.603, -17.603, -18.799, -19.761, -20.73, -21.701, -22.676, -23.653, -24.632, -25.613, -26.596, -27.581, -28.567, -29.558, -30.593, -31.66, -32.637, -33.595, -34.562, -35.548, -36.566, -37.602, -38.609, -39.601, -40.601, -41.607, -42.614, -43.597, -44.584, -45.586, -46.616, -47.601, -48.565, -49.604, -50.599, -51.594, -52.605, -53.611, -54.629, -55.621, -56.632, -57.621, -58.608, -59.608, -60.604, -61.133, -62.615, -63.607, -64.613, -65.604, -66.595, -67.592, -68.589, -69.5759999999999, -70.5809999999999, -71.589, -72.5459999999999, -73.5489999999999, -74.568, -75.533, -76.6149999999999, -77.5669999999999, -78.607, -79.613, -80.6259999999999, -81.6079999999999, -82.6019999999999, -83.5929999999999, -84.5939999999999, -85.601, -86.601, -87.598, -89.0699999999999, -91.086, -93.0849999999999, -95.0849999999999, -97.084, -99.084, -101.086, -103.087, -105.088, -107.089, -109.089, -111.09, -113.09, -115.09, -117.091, -119.091, -121.091, -123.092, -125.092, -127.092, -129.092, -131.093, -133.093, -135.093, -137.093, -140.065, -144.067, -148.068, -152.069, -156.07, -160.072, -164.073, -168.074, -172.075, -176.076, -180.076, -184.077, -188.078, -192.079, -196.079, -200.08, -204.08, -208.081, -212.082, -216.082, -220.082, -224.083, -228.083, -232.084, -236.084, -238.103, -248.103, -258.103, -268.103, -278.103, -288.103, -298.103, -308.103, -318.103, -328.103, -338.103, -348.103, -358.103, -368.103, -378.103, -388.103, -398.103, -408.103, -418.103, -428.103, -438.103, -448.103, -458.103, -468.103, -478.103, -488.103, -498.103, -508.103, -518.103, -528.103, -538.103, -548.103, -558.103, -568.103, -578.103, -588.103, -598.103, -608.103, -618.103, -628.103, -638.103, -648.103, -658.103, -668.103, -678.103, -688.103, -698.103, -708.103, -718.103, -728.103, -738.103, -748.103, -758.103, -768.103, -778.103, -788.103, -798.103, -808.103, -818.103, -828.103, -838.103, -848.103, -858.103, -868.103, -878.103, -888.103, -898.103, -908.103, -918.103, -928.103, -938.103, -948.103, -958.103, -968.103, -978.103, -988.103, -998.103, -1008.103, -1018.103, -1028.103, -1038.103, -1048.103, -1058.103, -1068.103, -1078.103, -1088.103, -1098.103, -1108.103, -1118.103, -1128.103, -1138.103, -1148.103, -1158.103, -1168.103, -1178.103, -1188.103, -1198.103, -1208.103, -1218.103, -1228.103, -1238.103, -1248.103, -1258.103, -1268.103, -1278.103, -1288.103, -1298.103, -1308.103, -1318.103, -1328.103, -1338.103, -1348.103, -1358.103, -1368.103, -1378.103, -1388.103, -1398.103, -1408.103, -1418.103, -1428.103, -1438.103, -1448.103, -1458.103, -1468.103, -1478.103, -1488.103, -1498.103, -1508.103, -1518.103, -1528.103, -1538.103, -1548.103, -1558.103, -1568.103, -1578.103, -1588.103, -1598.103, -1608.103, -1618.103, -1628.103, -1638.103, -1648.103, -1658.103, -1668.103, -1678.103, -1688.103, -1698.103, -1708.103, -1718.103, -1728.103, -1738.103, -1748.103, -1758.103, -1768.103, -1778.103, -1788.103, -1798.103, -1808.103, -1818.103, -1828.103, -1838.103, -1848.103, -1858.103, -1868.103, -1878.103, -1888.103, -1898.103, -1908.103, -1918.103, -1928.103, -1938.103, -1948.103, -1958.103, -1968.103, -1978.103, -1988.103, -1998.103, -2008.103, -2018.103, -2028.103, -2038.103, -2048.103, -2058.103, -2068.103, -2078.103, -2088.103, -2098.103, -2108.103, -2118.103, -2128.103, -2138.103, -2148.103, -2158.103, -2168.103, -2178.103, -2188.103, -2198.103, -2208.103, -2218.103, -2228.103, -2238.103, -2248.103, -2258.103, -2268.103, -2278.103, -2288.103], "Probability": [9.99999999999999e-08, 0.0006900000000000001, 0.00046, 0.00233, 0.00553, 0.00457, 0.02033, 0.01649, 0.03877, 0.038079999999999996, 0.06809, 0.11214, 0.10112, 0.24406, 0.32337000000000005, 0.40864, 0.68745, 0.66279, 0.51412, 0.6556100000000001, 0.54588, 0.37231000000000003, 0.25473, 0.16959, 0.11369, 0.16946999999999998, 0.10094, 0.05732, 0.02806, 0.013160000000000002, 0.00623, 0.0042, 0.0008, 0.00015, 0.0, 0.0, 0.0, 1.1999999999999999e-05, 0.000113, 0.0006560000000000001, 0.002567, 0.007149, 0.014804, 0.023583, 0.029715, 0.030307, 0.025527, 0.018080000000000002, 0.01107, 0.007377000000000001, 0.010637, 0.019095, 0.022178, 0.016434, 0.009037, 0.004989, 0.003978, 0.004124, 0.004152, 0.0039250000000000005, 0.003457, 0.003186, 0.0027010000000000003, 0.0027129999999999997, 0.002481, 0.002412, 0.001907, 0.001938, 0.0017599999999999998, 0.001575, 0.0015409999999999998, 0.001485, 0.001557, 0.001895, 0.002427, 0.003357, 0.004095, 0.004714, 0.005033999999999999, 0.005152, 0.005442000000000001, 0.005859, 0.006617, 0.0070940000000000005, 0.007404, 0.007164, 0.006563, 0.005620000000000001, 0.004691, 0.00368, 0.003049, 0.00221, 0.001928, 0.0017610000000000002, 0.0015300000000000001, 0.001215, 0.0013900000000000002, 0.001216, 0.0014219999999999999, 0.001384, 0.001368, 0.001316, 0.001153, 0.0010760000000000001, 0.000921, 0.0007570000000000001, 0.000696, 0.0006180000000000001, 0.00054, 0.00048300000000000003, 0.00043200000000000004, 0.000388, 0.00035150000000000003, 0.00031800000000000003, 0.000289, 0.000264, 0.00024150000000000002, 0.000222, 0.0002045, 0.000189, 0.00017500000000000003, 0.00016250000000000002, 0.000151, 0.000141, 0.0001315, 0.000123, 0.000115, 0.00010800000000000001, 0.0001015, 9.549999999999999e-05, 8.999999999999999e-05, 8.45e-05, 7.775e-05, 6.95e-05, 6.25e-05, 5.625e-05, 5.1000000000000006e-05, 4.625e-05, 4.225e-05, 3.85e-05, 3.525e-05, 3.25e-05, 3e-05, 2.775e-05, 2.5750000000000002e-05, 2.375e-05, 2.225e-05, 2.075e-05, 1.925e-05, 1.8e-05, 1.7e-05, 1.6000000000000003e-05, 1.5e-05, 1.4e-05, 1.325e-05, 1.25e-05, 1.1750000000000001e-05, 1.10288508e-05, 9.70207309e-06, 8.57766708e-06, 7.61863494e-06, 6.79579478e-06, 6.08592095e-06, 5.47037339e-06, 4.93407595e-06, 4.46474684e-06, 4.05231317e-06, 3.68846160e-06, 3.36629039e-06, 3.08003795e-06, 2.82486935e-06, 2.59670729e-06, 2.39209743e-06, 2.20810027e-06, 2.04220408e-06, 1.89225409e-06, 1.75639484e-06, 1.63302290e-06, 1.52074791e-06, 1.41836036e-06, 1.32480479e-06, 1.23915746e-06, 1.16060760e-06, 1.08844166e-06, 1.02203004e-06, 9.60815818e-07, 9.04305156e-07, 8.52059172e-07, 8.03686951e-07, 7.58839553e-07, 7.17204875e-07, 6.78503203e-07, 6.42483371e-07, 6.08919436e-07, 5.77607778e-07, 5.48364581e-07, 5.21023630e-07, 4.95434382e-07, 4.71460278e-07, 4.48977254e-07, 4.27872435e-07, 4.08042979e-07, 3.89395057e-07, 3.71842946e-07, 3.55308232e-07, 3.39719090e-07, 3.25009655e-07, 3.11119455e-07, 2.97992900e-07, 2.85578837e-07, 2.73830140e-07, 2.62703349e-07, 2.52158340e-07, 2.42158035e-07, 2.32668134e-07, 2.23656879e-07, 2.15094834e-07, 2.06954696e-07, 1.99211112e-07, 1.91840523e-07, 1.84821017e-07, 1.78132195e-07, 1.71755055e-07, 1.65671881e-07, 1.59866140e-07, 1.54322396e-07, 1.49026222e-07, 1.43964128e-07, 1.39123490e-07, 1.34492483e-07, 1.30060028e-07, 1.25815736e-07, 1.21749857e-07, 1.17853236e-07, 1.14117272e-07, 1.10533879e-07, 1.07095450e-07, 1.03794826e-07, 1.00625262e-07, 9.75804056e-08, 9.46542644e-08, 9.18411867e-08, 8.91358374e-08, 8.65331784e-08, 8.40284491e-08, 8.16171491e-08, 7.92950220e-08, 7.70580397e-08, 7.49023889e-08, 7.28244573e-08, 7.08208221e-08, 6.88882381e-08, 6.70236272e-08, 6.52240686e-08, 6.34867894e-08, 6.18091563e-08, 6.01886672e-08, 5.86229437e-08, 5.71097242e-08, 5.56468574e-08, 5.42322958e-08, 5.28640901e-08, 5.15403840e-08, 5.02594086e-08, 4.90194782e-08, 4.78189855e-08, 4.66563972e-08, 4.55302507e-08, 4.44391498e-08, 4.33817614e-08, 4.23568124e-08, 4.13630861e-08, 4.03994200e-08, 3.94647025e-08, 3.85578705e-08, 3.76779070e-08, 3.68238389e-08, 3.59947345e-08, 3.51897016e-08, 3.44078860e-08, 3.36484689e-08, 3.29106658e-08, 3.21937246e-08, 3.14969239e-08, 3.08195721e-08, 3.01610052e-08, 2.95205864e-08, 2.88977041e-08, 2.82917711e-08, 2.77022234e-08, 2.71285193e-08, 2.65701380e-08, 2.60265790e-08, 2.54973612e-08, 2.49820219e-08, 2.44801160e-08, 2.39912151e-08, 2.35149073e-08, 2.30507957e-08, 2.25984985e-08, 2.21576477e-08, 2.17278892e-08, 2.13088815e-08, 2.09002958e-08, 2.05018149e-08, 2.01131331e-08, 1.97339558e-08, 1.93639987e-08, 1.90029874e-08, 1.86506574e-08, 1.83067534e-08, 1.79710289e-08, 1.76432458e-08, 1.73231744e-08, 1.70105927e-08, 1.67052864e-08, 1.64070482e-08, 1.61156781e-08, 1.58309825e-08, 1.55527744e-08, 1.52808729e-08, 1.50151031e-08, 1.47552959e-08, 1.45012876e-08, 1.42529198e-08, 1.40100394e-08, 1.37724979e-08, 1.35401518e-08, 1.33128621e-08, 1.30904941e-08, 1.28729175e-08, 1.26600059e-08, 1.24516370e-08, 1.22476921e-08, 1.20480564e-08, 1.18526183e-08, 1.16612701e-08, 1.14739068e-08, 1.12904270e-08, 1.11107322e-08, 1.09347268e-08, 1.07623182e-08, 1.05934164e-08, 1.04279341e-08, 1.02657867e-08, 1.01068918e-08, 9.95116964e-09, 9.79854271e-09, 9.64893572e-09, 9.50227555e-09, 9.35849116e-09, 9.21751355e-09, 9.07927567e-09, 8.94371237e-09, 8.81076032e-09, 8.68035797e-09, 8.55244548e-09, 8.42696466e-09, 8.30385896e-09, 8.18307335e-09, 8.06455432e-09, 7.94824981e-09, 7.83410920e-09]} From 28ec76c6c977c89f8c980c7d4ec4265e6cb66b11 Mon Sep 17 00:00:00 2001 From: Juliana-S Date: Tue, 10 Aug 2021 21:45:43 -0400 Subject: [PATCH 117/199] fixed molecular final states file and added extended --- mermithid/misc/saenz_mfs.json | 2 +- mermithid/misc/saenz_mfs_ctd.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 mermithid/misc/saenz_mfs_ctd.json diff --git a/mermithid/misc/saenz_mfs.json b/mermithid/misc/saenz_mfs.json index 3d3b9d7d..89524096 100644 --- a/mermithid/misc/saenz_mfs.json +++ b/mermithid/misc/saenz_mfs.json @@ -1 +1 @@ -{"Binding energy": [1.897, 1.844, 1.773, 1.65, 1.546, 1.455, 1.341, 1.232, 1.138, 1.047, 0.96, 0.849, 0.754, 0.647999999999999, 0.538, 0.446, 0.345, 0.24, 0.151999999999999, 0.0629999999999999, -0.0429999999999999, -0.147, -0.247, -0.347, -0.446999999999999, -0.613, -0.865, -1.112, -1.36, -1.61, -1.86, -2.186, -2.68199999999999, -3.23499999999999, -3.75, -16.603, -17.603, -18.799, -19.761, -20.73, -21.701, -22.676, -23.653, -24.632, -25.613, -26.596, -27.581, -28.567, -29.558, -30.593, -31.66, -32.637, -33.595, -34.562, -35.548, -36.566, -37.602, -38.609, -39.601, -40.601, -41.607, -42.614, -43.597, -44.584, -45.586, -46.616, -47.601, -48.565, -49.604, -50.599, -51.594, -52.605, -53.611, -54.629, -55.621, -56.632, -57.621, -58.608, -59.608, -60.604, -61.133, -62.615, -63.607, -64.613, -65.604, -66.595, -67.592, -68.589, -69.5759999999999, -70.5809999999999, -71.589, -72.5459999999999, -73.5489999999999, -74.568, -75.533, -76.6149999999999, -77.5669999999999, -78.607, -79.613, -80.6259999999999, -81.6079999999999, -82.6019999999999, -83.5929999999999, -84.5939999999999, -85.601, -86.601, -87.598, -89.0699999999999, -91.086, -93.0849999999999, -95.0849999999999, -97.084, -99.084, -101.086, -103.087, -105.088, -107.089, -109.089, -111.09, -113.09, -115.09, -117.091, -119.091, -121.091, -123.092, -125.092, -127.092, -129.092, -131.093, -133.093, -135.093, -137.093, -140.065, -144.067, -148.068, -152.069, -156.07, -160.072, -164.073, -168.074, -172.075, -176.076, -180.076, -184.077, -188.078, -192.079, -196.079, -200.08, -204.08, -208.081, -212.082, -216.082, -220.082, -224.083, -228.083, -232.084, -236.084], "Probability": [9.99999999999999e-08, 0.0006900000000000001, 0.00046, 0.00233, 0.00553, 0.00457, 0.02033, 0.01649, 0.03877, 0.038079999999999996, 0.06809, 0.11214, 0.10112, 0.24406, 0.32337000000000005, 0.40864, 0.68745, 0.66279, 0.51412, 0.6556100000000001, 0.54588, 0.37231000000000003, 0.25473, 0.16959, 0.11369, 0.16946999999999998, 0.10094, 0.05732, 0.02806, 0.013160000000000002, 0.00623, 0.0042, 0.0008, 0.00015, 0.0, 0.0, 0.0, 1.1999999999999999e-05, 0.000113, 0.0006560000000000001, 0.002567, 0.007149, 0.014804, 0.023583, 0.029715, 0.030307, 0.025527, 0.018080000000000002, 0.01107, 0.007377000000000001, 0.010637, 0.019095, 0.022178, 0.016434, 0.009037, 0.004989, 0.003978, 0.004124, 0.004152, 0.0039250000000000005, 0.003457, 0.003186, 0.0027010000000000003, 0.0027129999999999997, 0.002481, 0.002412, 0.001907, 0.001938, 0.0017599999999999998, 0.001575, 0.0015409999999999998, 0.001485, 0.001557, 0.001895, 0.002427, 0.003357, 0.004095, 0.004714, 0.005033999999999999, 0.005152, 0.005442000000000001, 0.005859, 0.006617, 0.0070940000000000005, 0.007404, 0.007164, 0.006563, 0.005620000000000001, 0.004691, 0.00368, 0.003049, 0.00221, 0.001928, 0.0017610000000000002, 0.0015300000000000001, 0.001215, 0.0013900000000000002, 0.001216, 0.0014219999999999999, 0.001384, 0.001368, 0.001316, 0.001153, 0.0010760000000000001, 0.000921, 0.0007570000000000001, 0.000696, 0.0006180000000000001, 0.00054, 0.00048300000000000003, 0.00043200000000000004, 0.000388, 0.00035150000000000003, 0.00031800000000000003, 0.000289, 0.000264, 0.00024150000000000002, 0.000222, 0.0002045, 0.000189, 0.00017500000000000003, 0.00016250000000000002, 0.000151, 0.000141, 0.0001315, 0.000123, 0.000115, 0.00010800000000000001, 0.0001015, 9.549999999999999e-05, 8.999999999999999e-05, 8.45e-05, 7.775e-05, 6.95e-05, 6.25e-05, 5.625e-05, 5.1000000000000006e-05, 4.625e-05, 4.225e-05, 3.85e-05, 3.525e-05, 3.25e-05, 3e-05, 2.775e-05, 2.5750000000000002e-05, 2.375e-05, 2.225e-05, 2.075e-05, 1.925e-05, 1.8e-05, 1.7e-05, 1.6000000000000003e-05, 1.5e-05, 1.4e-05, 1.325e-05, 1.1750000000000001e-05, NaN]} \ No newline at end of file +{"Binding energy": [1.897, 1.844, 1.773, 1.65, 1.546, 1.455, 1.341, 1.232, 1.138, 1.047, 0.96, 0.849, 0.754, 0.647999999999999, 0.538, 0.446, 0.345, 0.24, 0.151999999999999, 0.0629999999999999, -0.0429999999999999, -0.147, -0.247, -0.347, -0.446999999999999, -0.613, -0.865, -1.112, -1.36, -1.61, -1.86, -2.186, -2.68199999999999, -3.23499999999999, -3.75, -16.603, -17.603, -18.799, -19.761, -20.73, -21.701, -22.676, -23.653, -24.632, -25.613, -26.596, -27.581, -28.567, -29.558, -30.593, -31.66, -32.637, -33.595, -34.562, -35.548, -36.566, -37.602, -38.609, -39.601, -40.601, -41.607, -42.614, -43.597, -44.584, -45.586, -46.616, -47.601, -48.565, -49.604, -50.599, -51.594, -52.605, -53.611, -54.629, -55.621, -56.632, -57.621, -58.608, -59.608, -60.604, -61.133, -62.615, -63.607, -64.613, -65.604, -66.595, -67.592, -68.589, -69.5759999999999, -70.5809999999999, -71.589, -72.5459999999999, -73.5489999999999, -74.568, -75.533, -76.6149999999999, -77.5669999999999, -78.607, -79.613, -80.6259999999999, -81.6079999999999, -82.6019999999999, -83.5929999999999, -84.5939999999999, -85.601, -86.601, -87.598, -89.0699999999999, -91.086, -93.0849999999999, -95.0849999999999, -97.084, -99.084, -101.086, -103.087, -105.088, -107.089, -109.089, -111.09, -113.09, -115.09, -117.091, -119.091, -121.091, -123.092, -125.092, -127.092, -129.092, -131.093, -133.093, -135.093, -137.093, -140.065, -144.067, -148.068, -152.069, -156.07, -160.072, -164.073, -168.074, -172.075, -176.076, -180.076, -184.077, -188.078, -192.079, -196.079, -200.08, -204.08, -208.081, -212.082, -216.082, -220.082, -224.083, -228.083, -232.084, -236.084], "Probability": [9.99999999999999e-08, 0.0006900000000000001, 0.00046, 0.00233, 0.00553, 0.00457, 0.02033, 0.01649, 0.03877, 0.038079999999999996, 0.06809, 0.11214, 0.10112, 0.24406, 0.32337000000000005, 0.40864, 0.68745, 0.66279, 0.51412, 0.6556100000000001, 0.54588, 0.37231000000000003, 0.25473, 0.16959, 0.11369, 0.16946999999999998, 0.10094, 0.05732, 0.02806, 0.013160000000000002, 0.00623, 0.0042, 0.0008, 0.00015, 0.0, 0.0, 0.0, 1.1999999999999999e-05, 0.000113, 0.0006560000000000001, 0.002567, 0.007149, 0.014804, 0.023583, 0.029715, 0.030307, 0.025527, 0.018080000000000002, 0.01107, 0.007377000000000001, 0.010637, 0.019095, 0.022178, 0.016434, 0.009037, 0.004989, 0.003978, 0.004124, 0.004152, 0.0039250000000000005, 0.003457, 0.003186, 0.0027010000000000003, 0.0027129999999999997, 0.002481, 0.002412, 0.001907, 0.001938, 0.0017599999999999998, 0.001575, 0.0015409999999999998, 0.001485, 0.001557, 0.001895, 0.002427, 0.003357, 0.004095, 0.004714, 0.005033999999999999, 0.005152, 0.005442000000000001, 0.005859, 0.006617, 0.0070940000000000005, 0.007404, 0.007164, 0.006563, 0.005620000000000001, 0.004691, 0.00368, 0.003049, 0.00221, 0.001928, 0.0017610000000000002, 0.0015300000000000001, 0.001215, 0.0013900000000000002, 0.001216, 0.0014219999999999999, 0.001384, 0.001368, 0.001316, 0.001153, 0.0010760000000000001, 0.000921, 0.0007570000000000001, 0.000696, 0.0006180000000000001, 0.00054, 0.00048300000000000003, 0.00043200000000000004, 0.000388, 0.00035150000000000003, 0.00031800000000000003, 0.000289, 0.000264, 0.00024150000000000002, 0.000222, 0.0002045, 0.000189, 0.00017500000000000003, 0.00016250000000000002, 0.000151, 0.000141, 0.0001315, 0.000123, 0.000115, 0.00010800000000000001, 0.0001015, 9.549999999999999e-05, 8.999999999999999e-05, 8.45e-05, 7.775e-05, 6.95e-05, 6.25e-05, 5.625e-05, 5.1000000000000006e-05, 4.625e-05, 4.225e-05, 3.85e-05, 3.525e-05, 3.25e-05, 3e-05, 2.775e-05, 2.5750000000000002e-05, 2.375e-05, 2.225e-05, 2.075e-05, 1.925e-05, 1.8e-05, 1.7e-05, 1.6000000000000003e-05, 1.5e-05, 1.4e-05, 1.325e-05, 1.25e-05, 1.1750000000000001e-05]} diff --git a/mermithid/misc/saenz_mfs_ctd.json b/mermithid/misc/saenz_mfs_ctd.json new file mode 100644 index 00000000..1c78c976 --- /dev/null +++ b/mermithid/misc/saenz_mfs_ctd.json @@ -0,0 +1 @@ +{"Binding energy": [1.897, 1.844, 1.773, 1.65, 1.546, 1.455, 1.341, 1.232, 1.138, 1.047, 0.96, 0.849, 0.754, 0.647999999999999, 0.538, 0.446, 0.345, 0.24, 0.151999999999999, 0.0629999999999999, -0.0429999999999999, -0.147, -0.247, -0.347, -0.446999999999999, -0.613, -0.865, -1.112, -1.36, -1.61, -1.86, -2.186, -2.68199999999999, -3.23499999999999, -3.75, -16.603, -17.603, -18.799, -19.761, -20.73, -21.701, -22.676, -23.653, -24.632, -25.613, -26.596, -27.581, -28.567, -29.558, -30.593, -31.66, -32.637, -33.595, -34.562, -35.548, -36.566, -37.602, -38.609, -39.601, -40.601, -41.607, -42.614, -43.597, -44.584, -45.586, -46.616, -47.601, -48.565, -49.604, -50.599, -51.594, -52.605, -53.611, -54.629, -55.621, -56.632, -57.621, -58.608, -59.608, -60.604, -61.133, -62.615, -63.607, -64.613, -65.604, -66.595, -67.592, -68.589, -69.5759999999999, -70.5809999999999, -71.589, -72.5459999999999, -73.5489999999999, -74.568, -75.533, -76.6149999999999, -77.5669999999999, -78.607, -79.613, -80.6259999999999, -81.6079999999999, -82.6019999999999, -83.5929999999999, -84.5939999999999, -85.601, -86.601, -87.598, -89.0699999999999, -91.086, -93.0849999999999, -95.0849999999999, -97.084, -99.084, -101.086, -103.087, -105.088, -107.089, -109.089, -111.09, -113.09, -115.09, -117.091, -119.091, -121.091, -123.092, -125.092, -127.092, -129.092, -131.093, -133.093, -135.093, -137.093, -140.065, -144.067, -148.068, -152.069, -156.07, -160.072, -164.073, -168.074, -172.075, -176.076, -180.076, -184.077, -188.078, -192.079, -196.079, -200.08, -204.08, -208.081, -212.082, -216.082, -220.082, -224.083, -228.083, -232.084, -236.084, -238.103, -248.103, -258.103, -268.103, -278.103, -288.103, -298.103, -308.103, -318.103, -328.103, -338.103, -348.103, -358.103, -368.103, -378.103, -388.103, -398.103, -408.103, -418.103, -428.103, -438.103, -448.103, -458.103, -468.103, -478.103, -488.103, -498.103, -508.103, -518.103, -528.103, -538.103, -548.103, -558.103, -568.103, -578.103, -588.103, -598.103, -608.103, -618.103, -628.103, -638.103, -648.103, -658.103, -668.103, -678.103, -688.103, -698.103, -708.103, -718.103, -728.103, -738.103, -748.103, -758.103, -768.103, -778.103, -788.103, -798.103, -808.103, -818.103, -828.103, -838.103, -848.103, -858.103, -868.103, -878.103, -888.103, -898.103, -908.103, -918.103, -928.103, -938.103, -948.103, -958.103, -968.103, -978.103, -988.103, -998.103, -1008.103, -1018.103, -1028.103, -1038.103, -1048.103, -1058.103, -1068.103, -1078.103, -1088.103, -1098.103, -1108.103, -1118.103, -1128.103, -1138.103, -1148.103, -1158.103, -1168.103, -1178.103, -1188.103, -1198.103, -1208.103, -1218.103, -1228.103, -1238.103, -1248.103, -1258.103, -1268.103, -1278.103, -1288.103, -1298.103, -1308.103, -1318.103, -1328.103, -1338.103, -1348.103, -1358.103, -1368.103, -1378.103, -1388.103, -1398.103, -1408.103, -1418.103, -1428.103, -1438.103, -1448.103, -1458.103, -1468.103, -1478.103, -1488.103, -1498.103, -1508.103, -1518.103, -1528.103, -1538.103, -1548.103, -1558.103, -1568.103, -1578.103, -1588.103, -1598.103, -1608.103, -1618.103, -1628.103, -1638.103, -1648.103, -1658.103, -1668.103, -1678.103, -1688.103, -1698.103, -1708.103, -1718.103, -1728.103, -1738.103, -1748.103, -1758.103, -1768.103, -1778.103, -1788.103, -1798.103, -1808.103, -1818.103, -1828.103, -1838.103, -1848.103, -1858.103, -1868.103, -1878.103, -1888.103, -1898.103, -1908.103, -1918.103, -1928.103, -1938.103, -1948.103, -1958.103, -1968.103, -1978.103, -1988.103, -1998.103, -2008.103, -2018.103, -2028.103, -2038.103, -2048.103, -2058.103, -2068.103, -2078.103, -2088.103, -2098.103, -2108.103, -2118.103, -2128.103, -2138.103, -2148.103, -2158.103, -2168.103, -2178.103, -2188.103, -2198.103, -2208.103, -2218.103, -2228.103, -2238.103, -2248.103, -2258.103, -2268.103, -2278.103, -2288.103], "Probability": [9.99999999999999e-08, 0.0006900000000000001, 0.00046, 0.00233, 0.00553, 0.00457, 0.02033, 0.01649, 0.03877, 0.038079999999999996, 0.06809, 0.11214, 0.10112, 0.24406, 0.32337000000000005, 0.40864, 0.68745, 0.66279, 0.51412, 0.6556100000000001, 0.54588, 0.37231000000000003, 0.25473, 0.16959, 0.11369, 0.16946999999999998, 0.10094, 0.05732, 0.02806, 0.013160000000000002, 0.00623, 0.0042, 0.0008, 0.00015, 0.0, 0.0, 0.0, 1.1999999999999999e-05, 0.000113, 0.0006560000000000001, 0.002567, 0.007149, 0.014804, 0.023583, 0.029715, 0.030307, 0.025527, 0.018080000000000002, 0.01107, 0.007377000000000001, 0.010637, 0.019095, 0.022178, 0.016434, 0.009037, 0.004989, 0.003978, 0.004124, 0.004152, 0.0039250000000000005, 0.003457, 0.003186, 0.0027010000000000003, 0.0027129999999999997, 0.002481, 0.002412, 0.001907, 0.001938, 0.0017599999999999998, 0.001575, 0.0015409999999999998, 0.001485, 0.001557, 0.001895, 0.002427, 0.003357, 0.004095, 0.004714, 0.005033999999999999, 0.005152, 0.005442000000000001, 0.005859, 0.006617, 0.0070940000000000005, 0.007404, 0.007164, 0.006563, 0.005620000000000001, 0.004691, 0.00368, 0.003049, 0.00221, 0.001928, 0.0017610000000000002, 0.0015300000000000001, 0.001215, 0.0013900000000000002, 0.001216, 0.0014219999999999999, 0.001384, 0.001368, 0.001316, 0.001153, 0.0010760000000000001, 0.000921, 0.0007570000000000001, 0.000696, 0.0006180000000000001, 0.00054, 0.00048300000000000003, 0.00043200000000000004, 0.000388, 0.00035150000000000003, 0.00031800000000000003, 0.000289, 0.000264, 0.00024150000000000002, 0.000222, 0.0002045, 0.000189, 0.00017500000000000003, 0.00016250000000000002, 0.000151, 0.000141, 0.0001315, 0.000123, 0.000115, 0.00010800000000000001, 0.0001015, 9.549999999999999e-05, 8.999999999999999e-05, 8.45e-05, 7.775e-05, 6.95e-05, 6.25e-05, 5.625e-05, 5.1000000000000006e-05, 4.625e-05, 4.225e-05, 3.85e-05, 3.525e-05, 3.25e-05, 3e-05, 2.775e-05, 2.5750000000000002e-05, 2.375e-05, 2.225e-05, 2.075e-05, 1.925e-05, 1.8e-05, 1.7e-05, 1.6000000000000003e-05, 1.5e-05, 1.4e-05, 1.325e-05, 1.25e-05, 1.1750000000000001e-05, 1.10288508e-05, 9.70207309e-06, 8.57766708e-06, 7.61863494e-06, 6.79579478e-06, 6.08592095e-06, 5.47037339e-06, 4.93407595e-06, 4.46474684e-06, 4.05231317e-06, 3.68846160e-06, 3.36629039e-06, 3.08003795e-06, 2.82486935e-06, 2.59670729e-06, 2.39209743e-06, 2.20810027e-06, 2.04220408e-06, 1.89225409e-06, 1.75639484e-06, 1.63302290e-06, 1.52074791e-06, 1.41836036e-06, 1.32480479e-06, 1.23915746e-06, 1.16060760e-06, 1.08844166e-06, 1.02203004e-06, 9.60815818e-07, 9.04305156e-07, 8.52059172e-07, 8.03686951e-07, 7.58839553e-07, 7.17204875e-07, 6.78503203e-07, 6.42483371e-07, 6.08919436e-07, 5.77607778e-07, 5.48364581e-07, 5.21023630e-07, 4.95434382e-07, 4.71460278e-07, 4.48977254e-07, 4.27872435e-07, 4.08042979e-07, 3.89395057e-07, 3.71842946e-07, 3.55308232e-07, 3.39719090e-07, 3.25009655e-07, 3.11119455e-07, 2.97992900e-07, 2.85578837e-07, 2.73830140e-07, 2.62703349e-07, 2.52158340e-07, 2.42158035e-07, 2.32668134e-07, 2.23656879e-07, 2.15094834e-07, 2.06954696e-07, 1.99211112e-07, 1.91840523e-07, 1.84821017e-07, 1.78132195e-07, 1.71755055e-07, 1.65671881e-07, 1.59866140e-07, 1.54322396e-07, 1.49026222e-07, 1.43964128e-07, 1.39123490e-07, 1.34492483e-07, 1.30060028e-07, 1.25815736e-07, 1.21749857e-07, 1.17853236e-07, 1.14117272e-07, 1.10533879e-07, 1.07095450e-07, 1.03794826e-07, 1.00625262e-07, 9.75804056e-08, 9.46542644e-08, 9.18411867e-08, 8.91358374e-08, 8.65331784e-08, 8.40284491e-08, 8.16171491e-08, 7.92950220e-08, 7.70580397e-08, 7.49023889e-08, 7.28244573e-08, 7.08208221e-08, 6.88882381e-08, 6.70236272e-08, 6.52240686e-08, 6.34867894e-08, 6.18091563e-08, 6.01886672e-08, 5.86229437e-08, 5.71097242e-08, 5.56468574e-08, 5.42322958e-08, 5.28640901e-08, 5.15403840e-08, 5.02594086e-08, 4.90194782e-08, 4.78189855e-08, 4.66563972e-08, 4.55302507e-08, 4.44391498e-08, 4.33817614e-08, 4.23568124e-08, 4.13630861e-08, 4.03994200e-08, 3.94647025e-08, 3.85578705e-08, 3.76779070e-08, 3.68238389e-08, 3.59947345e-08, 3.51897016e-08, 3.44078860e-08, 3.36484689e-08, 3.29106658e-08, 3.21937246e-08, 3.14969239e-08, 3.08195721e-08, 3.01610052e-08, 2.95205864e-08, 2.88977041e-08, 2.82917711e-08, 2.77022234e-08, 2.71285193e-08, 2.65701380e-08, 2.60265790e-08, 2.54973612e-08, 2.49820219e-08, 2.44801160e-08, 2.39912151e-08, 2.35149073e-08, 2.30507957e-08, 2.25984985e-08, 2.21576477e-08, 2.17278892e-08, 2.13088815e-08, 2.09002958e-08, 2.05018149e-08, 2.01131331e-08, 1.97339558e-08, 1.93639987e-08, 1.90029874e-08, 1.86506574e-08, 1.83067534e-08, 1.79710289e-08, 1.76432458e-08, 1.73231744e-08, 1.70105927e-08, 1.67052864e-08, 1.64070482e-08, 1.61156781e-08, 1.58309825e-08, 1.55527744e-08, 1.52808729e-08, 1.50151031e-08, 1.47552959e-08, 1.45012876e-08, 1.42529198e-08, 1.40100394e-08, 1.37724979e-08, 1.35401518e-08, 1.33128621e-08, 1.30904941e-08, 1.28729175e-08, 1.26600059e-08, 1.24516370e-08, 1.22476921e-08, 1.20480564e-08, 1.18526183e-08, 1.16612701e-08, 1.14739068e-08, 1.12904270e-08, 1.11107322e-08, 1.09347268e-08, 1.07623182e-08, 1.05934164e-08, 1.04279341e-08, 1.02657867e-08, 1.01068918e-08, 9.95116964e-09, 9.79854271e-09, 9.64893572e-09, 9.50227555e-09, 9.35849116e-09, 9.21751355e-09, 9.07927567e-09, 8.94371237e-09, 8.81076032e-09, 8.68035797e-09, 8.55244548e-09, 8.42696466e-09, 8.30385896e-09, 8.18307335e-09, 8.06455432e-09, 7.94824981e-09, 7.83410920e-09]} From 3a4c198bfffba38f570224926d84fb91ab988a03 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 11 Aug 2021 16:44:43 +0200 Subject: [PATCH 118/199] removed option configuration via fit_options dictionary in DoOneFit, GetPDF, and GenAndFit. --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index fde12c03..115eb058 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -322,7 +322,7 @@ def InternalConfigure(self, config_dict): def ConfigureFit(self): # configure fit - energy_limits = [max([self.endpoint-300, np.min(self.energies)+3.5]), min([self.endpoint+300, np.max(self.energies)-3.5])] + energy_limits = [max([self.endpoint-1000, np.min(self.energies)+3.5]), min([self.endpoint+1000, np.max(self.energies)-3.5])] neutrino_limits = [-(np.max(self.energies) - energy_limits[1]-1)**2, (energy_limits[0]-np.min(self.energies)-1)**2] @@ -1272,9 +1272,18 @@ def Efficiency(self, E, freq=False, pseudo=False): def GenAndFit(params, counts, fit_config_dict,fit_options, sampled_priors, i, fixed_data = [], error_scaling=1, tilt = None, fit_tilt = False, event=None): - scattered = fit_options['scattered'] - distorted = fit_options['distorted'] - fit_nu_mass = fit_options['fit_nu_mass'] + """if 'scattered' in fit_options.keys(): + scattered = fit_options['scattered'] + else: + scattered = None + if 'distorted' in fit_options.keys(): + distorted = fit_options['distorted'] + else: + distorted = None + if 'fit_nu_mass' in fit_options.keys(): + fit_nu_mass = fit_options['fit_nu_mass'] + else: + fit_nu_mass = None""" if i%20 == 0: logger.info('Sampling: {}'.format(i)) @@ -1285,8 +1294,8 @@ def GenAndFit(params, counts, fit_config_dict,fit_options, sampled_priors, T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) - T.is_scattered = scattered - T.is_distorted = distorted + #T.is_scattered = scattered + #T.is_distorted = distorted T.error_scaling = error_scaling T.integrate_bins = True @@ -1304,10 +1313,10 @@ def GenAndFit(params, counts, fit_config_dict,fit_options, sampled_priors, _, new_data = T.GenerateAsimovData(params) - if fit_nu_mass: - T.fix_nu_mass = False + #if fit_nu_mass: + # T.fix_nu_mass = False - elif fit_tilt: + if fit_tilt: T.fix_tilt = False T.freq_data = new_data @@ -1327,15 +1336,15 @@ def GenAndFit(params, counts, fit_config_dict,fit_options, sampled_priors, def DoOneFit(data, fit_config_dict, fit_options, sampled_parameters={}, error_scaling=0, tilt=None, fit_tilt = False, data_is_energy=False): - scattered = fit_options['scattered'] + """scattered = fit_options['scattered'] distorted = fit_options['distorted'] - fit_nu_mass = fit_options['fit_nu_mass'] + fit_nu_mass = fit_options['fit_nu_mass']""" #print('do one fit', tilt, fit_tilt) T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) - T.is_scattered = scattered - T.is_distorted = distorted + #T.is_scattered = scattered + #T.is_distorted = distorted T.error_scaling = error_scaling T.integrate_bins = True logger.info('Energy stepsize: {}'.format(T.denergy)) @@ -1344,10 +1353,10 @@ def DoOneFit(data, fit_config_dict, fit_options, sampled_parameters={}, error_sc T.tilted_efficiency = True T.tilt = tilt - if fit_nu_mass: - logger.info('Fitting neutrino mass') - T.fix_nu_mass = False - elif fit_tilt: + #if fit_nu_mass: + # logger.info('Fitting neutrino mass') + # T.fix_nu_mass = False + if fit_tilt: logger.info('Going to fit efficiency tilt') T.fix_tilt = False @@ -1364,14 +1373,14 @@ def GetPDF(fit_config_dict, fit_options, params, plot=False): logger.info('Plotting lineshape: {}'.format(plot)) logger.info('PDF for params: {}'.format(params)) logger.info(fit_options) - scattered = fit_options['scattered'] - distorted = fit_options['distorted'] + #scattered = fit_options['scattered'] + #distorted = fit_options['distorted'] T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) T.plot_lineshape = plot - T.is_scatterd=scattered - T.is_distorted=distorted + #T.is_scatterd=scattered + #T.is_distorted=distorted pdf = T.TritiumSpectrumBackground(T.energies, *params) _, asimov_binned_data = T.GenerateAsimovData(params) From 54a0516c3b5a7e35f2a076c391714c1e14e75daa Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Sun, 15 Aug 2021 13:12:13 -0400 Subject: [PATCH 119/199] Avoid flipping lineshape for simplified model --- mermithid/misc/FakeTritiumDataFunctions.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index e96bdbdd..6a39f766 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -316,15 +316,16 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) #Convolving - if (lineshape=='detailed_scattering' or lineshape=='detailed') and (resolution_function == 'simulated_resolution' or resolution_function == 'simulated'): - convolved_list = [] - for j in range(len(lineshape_rates)): - beta_rates = spectral_rate(K_segments[j], Q, mnu, final_state_array) - convolved_list.append(convolve(beta_rates, lineshape_rates[j], mode='same')) - convolved = np.concatenate(convolved_list, axis=None) - - else: - lineshape_rates = np.flipud(lineshape_rates) + if (lineshape=='detailed_scattering' or lineshape=='detailed'): + if (resolution_function == 'simulated_resolution' or resolution_function == 'simulated'): + convolved_list = [] + for j in range(len(lineshape_rates)): + beta_rates = spectral_rate(K_segments[j], Q, mnu, final_state_array) + convolved_list.append(convolve(beta_rates, lineshape_rates[j], mode='same')) + convolved = np.concatenate(convolved_list, axis=None) + else: + lineshape_rates = np.flipud(lineshape_rates) + else: beta_rates = spectral_rate(K, Q, mnu, final_state_array) convolved = convolve(beta_rates, lineshape_rates, mode='same') From c606dae841f945316cbcb5776009d30544ff2af5 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 23 Aug 2021 10:22:59 -0400 Subject: [PATCH 120/199] rearrange the configurations in complex line shape fitter test --- test_analysis/Complex_line_shape_fitter.py | 38 ++++++++++++---------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index e2cbfdfd..2f8d8a38 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -22,7 +22,7 @@ def test_complex_lineshape(self): reader_config = { "action": "read", - "filename": "/host/shallow_trap_high_stats_above_10600_channel_a_concat.root", + "filename": "/host/october_2019_kr_calibration_channel_b_merged.root", "object_type": "TMultiTrackEventData", "object_name": "multiTrackEvents:Event", "use_katydid": False, @@ -31,8 +31,23 @@ def test_complex_lineshape(self): complexLineShape_config = { 'bins_choice': np.linspace(0e6, 100e6, 1000), - 'gases': ["H2", "He"], # Ar, Kr + 'gases': ["H2", "He", "Ar", "Kr"], # Ar, Kr 'max_scatters': 20, + # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, 'simulated_resolution_scaled_fit_scatter_peak_ratio', 'gaussian_resolution_fit_scatter_peak_ratio' + 'resolution_function': 'simulated_resolution_scaled_fit_scatter_peak_ratio', + #choose the parameters you want to fix from ['B field','amplitude','width scale factor', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] plus the gas scatter fractions as ['H2 scatter fraction'], + 'fixed_parameter_names': ['survival probability', 'width scale factor', 'H2 scatter fraction', 'He scatter fraction', 'Ar scatter fraction'], + 'fixed_parameter_values': [1.0, 1.0, 0.817, 0.07, 0.08], + # This is an important parameter which determines how finely resolved + # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown + 'num_points_in_std_array': 4000, + 'RF_ROI_MIN': 25859375000.0, + # shake_spectrum_parameters.json and oscillator strength data can be found at https://github.com/project8/scripts/tree/master/yuhao/line_shape_fitting/data + 'shake_spectrum_parameters_json_path': '../mermithid/misc/shake_spectrum_parameters.json', + 'path_to_osc_strengths_files': '/host/', + 'path_to_scatter_spectra_file': '/host/', + 'path_to_ins_resolution_data_txt': '/host/October_FTC_resolution/all_res_cf14.400.txt' + 'fixed_scatter_proportion': True, # When fixed_scatter_proportion is True, set the scatter proportion for the gases below 'gas_scatter_proportion': [0.8, 0.2],#0.827, 0.076, 0.068, 0.028 # 0.75, 0.25 @@ -43,8 +58,6 @@ def test_complex_lineshape(self): 'fixed_survival_probability': False, # When option fixed_survival_probability is True, assign the survival probability below 'survival_prob': 15/16., # assuming total cross section for elastic scattering is 1/10 of inelastic scattering - # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, 'simulated_resolution_scaled_fit_scatter_peak_ratio', 'gaussian_resolution_fit_scatter_peak_ratio' - 'resolution_function': 'gaussian_resolution_fit_scatter_peak_ratio', # specific choice of parameters in the gaussian lorentzian composite resolution function 'recon_eff_param_a': 0.005569990343215976, 'recon_eff_param_b': 0.351, @@ -56,19 +69,8 @@ def test_complex_lineshape(self): 'A_array': [0.076, 0.341, 0.381, 0.203], #parameter for simulated resolution scaled resolution 'fit_recon_eff': False, - #parameters for simulated resolution scaled with scatter peak ratio fitted - #choose the parameters you want to fix from ['B field','amplitude','width scale factor', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] plus the gas scatter fractions as ['H2 scatter fraction'], - 'fixed_parameter_names': ['survival probability', 'H2 scatter fraction'], - 'fixed_parameter_values': [1.0, 0.896], - # This is an important parameter which determines how finely resolved - # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown - 'num_points_in_std_array': 10000, - 'RF_ROI_MIN': 25850000000.0, - # shake_spectrum_parameters.json and oscillator strength data can be found at https://github.com/project8/scripts/tree/master/yuhao/line_shape_fitting/data - 'shake_spectrum_parameters_json_path': '../mermithid/misc/shake_spectrum_parameters.json', - 'path_to_osc_strengths_files': '/host/', - 'path_to_scatter_spectra_file': '/host/', - 'path_to_ins_resolution_data_txt': '/host/res_all_conversion_max15.5_alltraps.txt' + #parameters for simulated resolution scaled with scatter peak ratio fitted + } b = IOCicadaProcessor("reader") @@ -105,7 +107,7 @@ def test_complex_lineshape(self): plot_title = 'fit ftc march with gases: {},\n scatter proportion: {},\n resolution function: {},\n sigma_array: {},\n A_array: {},\n'.format(complexLineShape_config['gases'], complexLineShape_config['gas_scatter_proportion'], complexLineShape_config['resolution_function'], complexLineShape_config['sigma_array'], complexLineShape_config['A_array']) plt.title(plot_title) plt.tight_layout() - plt.savefig('/host/plots/fit_shallow_trap_above_10600_with_gaussian_resolution.png'.format(len(complexLineShape_config['gases']))) + plt.savefig('/host/plots/fit_october_ftc_simulated_resolution.png') if __name__ == '__main__': From ba8410b8bb9083a65ef03f981d253aa298829d30 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Mon, 23 Aug 2021 10:35:24 -0400 Subject: [PATCH 121/199] rearrange the configurations in complex line shape fitter test --- test_analysis/Complex_line_shape_fitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 2f8d8a38..25916376 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -46,7 +46,7 @@ def test_complex_lineshape(self): 'shake_spectrum_parameters_json_path': '../mermithid/misc/shake_spectrum_parameters.json', 'path_to_osc_strengths_files': '/host/', 'path_to_scatter_spectra_file': '/host/', - 'path_to_ins_resolution_data_txt': '/host/October_FTC_resolution/all_res_cf14.400.txt' + 'path_to_ins_resolution_data_txt': '/host/October_FTC_resolution/all_res_cf14.400.txt', 'fixed_scatter_proportion': True, # When fixed_scatter_proportion is True, set the scatter proportion for the gases below From 8aac6ff65380170c8ec82faed70865898ce86d2f Mon Sep 17 00:00:00 2001 From: cclaessens Date: Tue, 21 Sep 2021 17:12:56 -0700 Subject: [PATCH 122/199] implemented correlated b and c sampling, limited resolution to greater 30, allowed failed fits to re-try a small number of times --- .../Fitters/MCUncertaintyPropagation.py | 37 +++- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 161 +++++++++++------- 2 files changed, 124 insertions(+), 74 deletions(-) diff --git a/mermithid/processors/Fitters/MCUncertaintyPropagation.py b/mermithid/processors/Fitters/MCUncertaintyPropagation.py index 57e1c518..ef72eeb8 100644 --- a/mermithid/processors/Fitters/MCUncertaintyPropagation.py +++ b/mermithid/processors/Fitters/MCUncertaintyPropagation.py @@ -67,9 +67,20 @@ def InitialFit(self): ''' - self.fitted_params, self.fitted_params_errors, self.Counts = self.fit(self.data, - self.fit_config_dict, - self.fit_options) + fit_successful = False + counter = 0 + while (not fit_successful) and counter < 15: + counter += 1 + try: + + self.fitted_params, self.fitted_params_errors, self.Counts = self.fit(self.data, + self.fit_config_dict, + self.fit_options) + fit_successful = True + except Exception as e: + print(e) + logger.error('Repeating fit') + continue logger.info('Best fit: {}'.format(self.fitted_params)) @@ -108,11 +119,21 @@ def ParameterSampling(self): # sequential sampling all_fit_returns = [] for i in range(start_j, self.N): - all_fit_returns.append(self.gen_and_fit(self.fitted_params, self.Counts, - self.fit_config_dict, - self.fit_options, - parameter_sampling[k_i], - i, fixed_data)) + fit_successful = False + counter = 0 + while (not fit_successful) and counter < 15: + counter += 1 + try: + all_fit_returns.append(self.gen_and_fit(self.fitted_params, self.Counts, + self.fit_config_dict, + self.fit_options, + parameter_sampling[k_i], + i, fixed_data)) + fit_successful = True + except Exception as e: + print(e) + logger.error('Repeating fit') + continue diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 115eb058..b828603c 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -93,7 +93,9 @@ def InternalConfigure(self, config_dict): # final state spectrum self.use_final_states = reader.read_param(config_dict, 'use_final_states', False) - self.final_state_array = reader.read_param(config_dict, 'final_state_array', [[0], [1]]) + self.final_state_array = reader.read_param(config_dict, 'final_states_array', [[0], [1]]) + + #logger.info('final state array: {}'.format(self.final_state_array)) # detector response @@ -113,6 +115,23 @@ def InternalConfigure(self, config_dict): self.scatter_peak_ratio_mean = reader.read_param(config_dict, 'scatter_peak_ratio_mean', 0.5) self.scatter_peak_ratio_width = reader.read_param(config_dict, 'scatter_peak_ratio_width', 0.1) + self.scale_mean = reader.read_param(config_dict, 'scale_mean', 1) + self.scale_width = reader.read_param(config_dict, 'scale_width', 0) + + + #Adding correlated parameters (will later incorporate into morpho processor) + self.b_c_corr = reader.read_param(config_dict, 'b_c_corr', 0) + self.b_scale_corr = reader.read_param(config_dict, 'b_scale_corr', 0) + self.c_scale_corr = reader.read_param(config_dict, 'c_scale_corr', 0) + self.correlated_b_c_scale = reader.read_param(config_dict, 'correlated_b_c_scale', False) + + stds = [self.scatter_peak_ratio_b_width, self.scatter_peak_ratio_c_width, self.scale_width] + self.b_c_scale_cov_matrix = [[stds[0]**2, self.b_c_corr*stds[0]*stds[1], self.b_scale_corr*stds[0]*stds[2]], + [self.b_c_corr*stds[1]*stds[0], stds[1]**2, self.c_scale_corr*stds[1]*stds[2]], + [self.b_scale_corr*stds[2]*stds[0], self.c_scale_corr*stds[2]*stds[1], stds[2]**2]] + + + self.res_mean = reader.read_param(config_dict, 'gaussian_resolution_mean', 15.0) self.res_width = reader.read_param(config_dict, 'gaussian_resolution_width', 1.0) @@ -134,7 +153,7 @@ def InternalConfigure(self, config_dict): self.constrained_parameters = reader.read_param(config_dict, 'constrained_parameters', []) self.constrained_means = reader.read_param(config_dict, 'constrained_means', []) self.constrained_widths = reader.read_param(config_dict, 'constrained_widths', []) - if len(self.constrained_parameter_names) > 0: + if len(self.constrained_parameters) > 0: logger.warning('Some parameters are constrained: {} - {}'.format(self.constrained_parameters, self.constrained_parameter_names)) #self.print_level = 1 @@ -168,6 +187,8 @@ def InternalConfigure(self, config_dict): snr_efficiency_dict['good_fit_index'] = [True]*len((snr_efficiency_dict['frequencies'])) snr_efficiency_dict['tritium_rates'] = snr_efficiency_dict['eff interp with slope correction'] snr_efficiency_dict['tritium_rates_error'] = snr_efficiency_dict['error interp with slope correction'] + #snr_efficiency_dict['tritium_rates'] = snr_efficiency_dict['eff interp no energy correction'] + #snr_efficiency_dict['tritium_rates_error'] = snr_efficiency_dict['error interp no energy correction'] if self.max_frequency == None: self.max_frequency = np.max(snr_efficiency_dict['frequency']) @@ -252,16 +273,22 @@ def InternalConfigure(self, config_dict): self.B = self.B_mean self.res = self.res_mean + if self.res <= 30.01/float(2*np.sqrt(2*np.log(2))): + logger.warning('Resolution small for shallow trap model. Setting to {}'.format(30.01/float(2*np.sqrt(2*np.log(2))))) + self.res = 30.01/float(2*np.sqrt(2*np.log(2))) + self.endpoint=reader.read_param(config_dict, 'true_endpoint', 18.573e3) self.two_gaussian_sig_1 = self.two_gaussian_sig_1_mean self.two_gaussian_sig_2 = self.two_gaussian_sig_2_mean - - if self.use_fixed_scatter_peak_ratio: - self.scatter_peak_ratio_b = self.scatter_peak_ratio_mean - self.scatter_peak_ratio_c = 1 - else: - self.scatter_peak_ratio_b = self.scatter_peak_ratio_b_mean - self.scatter_peak_ratio_c = self.scatter_peak_ratio_c_mean + self.width_scaling = self.scale_mean + + #if self.use_fixed_scatter_peak_ratio: + # self.scatter_peak_ratio_b = self.scatter_peak_ratio_mean + # self.scatter_peak_ratio_c = 1 + #else: + self.scatter_peak_ratio_b = self.scatter_peak_ratio_b_mean + self.scatter_peak_ratio_c = self.scatter_peak_ratio_c_mean + self.parameter_samples = {} ######################################### @@ -323,7 +350,9 @@ def InternalConfigure(self, config_dict): def ConfigureFit(self): # configure fit energy_limits = [max([self.endpoint-1000, np.min(self.energies)+3.5]), min([self.endpoint+1000, np.max(self.energies)-3.5])] - neutrino_limits = [-(np.max(self.energies) - energy_limits[1]-1)**2, (energy_limits[0]-np.min(self.energies)-1)**2] + #neutrino_limits = [-(np.max(self.energies) - energy_limits[1]-1)**2, (energy_limits[0]-np.min(self.energies)-1)**2] + neutrino_limits = [-300**2, 300**2] + #logger.warning('Neutrino mass limited to: {} - {}'.format(*np.sqrt(np.array(neutrino_limits)))) #logger.warning('Neutrino mass fitted: {}'.format(self.fit_nu_mass)) @@ -365,9 +394,9 @@ def ConfigureFit(self): logger.info('Initial values: {}'.format(self.initial_values)) if len(self.constrained_parameters) > 0: - #logger.info('{}\n{}'.format(self.parameter_names, self.constrained_parameter_names)) + #logger.info('{}\n{}'.format(self.constrained_parameters, self.parameter_names)) #self.constrained_parameters = [self.parameter_names.index(p) for p in self.constrained_parameter_names[0]] - self.print_level=1 + self.print_level=0 @@ -497,7 +526,6 @@ def GenerateData(self, params, N): pdf = np.float64(np.longdouble(pdf/np.sum(pdf))) - self.data = np.random.choice(x, np.random.poisson(N), p=pdf/np.sum(pdf)) self.freq_data = self.Frequency(self.data) @@ -548,11 +576,13 @@ def ConvertAndHistogram(self, weights=None): def SamplePriors(self, sampled_parameters): logger.info('Sampling: {}'.format([k for k in sampled_parameters.keys() if sampled_parameters[k]])) - self.parameter_samples = {} sample_values = [] if 'res' in sampled_parameters.keys() and sampled_parameters['res']: self.res = self.Gaussian_sample(self.res_mean, self.res_width) + if self.res <= 30.01/float(2*np.sqrt(2*np.log(2))): + logger.warning('Sampled resolution small. Setting to {}'.format(30.01/float(2*np.sqrt(2*np.log(2))))) + self.res = 30.01/float(2*np.sqrt(2*np.log(2))) self.parameter_samples['res'] = self.res sample_values.append(self.res) if 'two_gaussian_std_1' in sampled_parameters.keys() and sampled_parameters['two_gaussian_std_1']: @@ -570,16 +600,36 @@ def SamplePriors(self, sampled_parameters): self.fix_scatter_ratio_c = True self.parameter_samples['scatter_peak_ratio'] = self.scatter_peak_ratio_b sample_values.append(self.scatter_peak_ratio_b) - if 'scatter_peak_ratio_b' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_b']: - self.scatter_peak_ratio_b = self.Gamma_sample(self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_b_width) + + if self.correlated_b_c_scale and 'scatter_peak_ratio_b' in sampled_parameters.keys() and 'scatter_peak_ratio_c' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_b'] and sampled_parameters['scatter_peak_ratio_c']: + logger.info('Correlated b, c, scale sampling') + correlated_vars = np.random.multivariate_normal([self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_c_mean, self.scale_mean], self.b_c_scale_cov_matrix) + self.scatter_peak_ratio_b = correlated_vars[0] + self.scatter_peak_ratio_c = correlated_vars[1] + self.width_scaling = correlated_vars[2] + self.fix_scatter_ratio_b = True self.parameter_samples['scatter_peak_ratio_b'] = self.scatter_peak_ratio_b sample_values.append(self.scatter_peak_ratio_b) - if 'scatter_peak_ratio_c' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_c']: - self.scatter_peak_ratio_c = self.Gamma_sample(self.scatter_peak_ratio_c_mean, self.scatter_peak_ratio_c_width) + self.parameter_samples['scatter_peak_ratio_c'] = self.scatter_peak_ratio_c sample_values.append(self.scatter_peak_ratio_c) self.fix_scatter_ratio_c = True + + else: + + if 'scatter_peak_ratio_b' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_b']: + logger.info('Uncorrelated b, c, scale sampling') + self.scatter_peak_ratio_b = self.Gamma_sample(self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_b_width) + self.fix_scatter_ratio_b = True + self.parameter_samples['scatter_peak_ratio_b'] = self.scatter_peak_ratio_b + sample_values.append(self.scatter_peak_ratio_b) + if 'scatter_peak_ratio_c' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_c']: + self.scatter_peak_ratio_c = self.Gamma_sample(self.scatter_peak_ratio_c_mean, self.scatter_peak_ratio_c_width) + self.parameter_samples['scatter_peak_ratio_c'] = self.scatter_peak_ratio_c + sample_values.append(self.scatter_peak_ratio_c) + self.fix_scatter_ratio_c = True + if 'B' in sampled_parameters.keys() and sampled_parameters['B']: self.B = self.Gaussian_sample(self.B_mean, self.B_width) self.parameter_samples['B'] = self.B @@ -640,26 +690,25 @@ def gauss_resolution_f(self, energy_array, A, sigma, mu): return f - def approximate_shape(self, K, Q, m_nu, index): - """ m_nu is neutrino mass squared """ + def approximate_shape(self, K, Q, m_nu_squared, index): shape = np.zeros(len(K)) - nu_mass_shape = ((Q - K[index])**2 -m_nu)**0.5 + nu_mass_shape = ((Q - K[index])**2 -m_nu_squared)**0.5 shape[index] = (Q - K[index])*nu_mass_shape return shape - def approximate_spectrum(self, E, Q, m_nu=0): + def approximate_spectrum(self, E, Q, m_nu_squared=0): """ model as in mermithid fake data generator: https://github.com/project8/mermithid/blob/feature/phase2-analysis/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py - but the ephasespace is approximate (some factors neglected) + but the ephasespace is approximate and neutrino parameter is mass squared (some factors neglected) """ # mnu is used in heaviside function - if m_nu >=0: - mnu = np.abs(m_nu**0.5) - else: - mnu = 0 + #if m_nu_squared >=0: + mnu = np.abs(m_nu_squared)**0.5 + #else: + # mnu = 0 if self.use_final_states: if isinstance(E, list) or isinstance(E, np.ndarray): @@ -667,9 +716,9 @@ def approximate_spectrum(self, E, Q, m_nu=0): Q_states = Q+self.final_state_array[0]-np.max(self.final_state_array[0]) approximate_e_phase_space = self.ephasespace(E, Q) - - index = [np.where(E < Q_states[i]-mnu) for i in range(N_states)] - beta_rates_array = [self.approximate_shape(E, Q_states[i], m_nu, index[i]) + index = [np.where(((Q_states[i]-E)**2-m_nu_squared > 0) & (Q_states[i]-E > 0)) for i in range(N_states)] + #index = [np.where(E < Q_states[i] -mnu) for i in range(N_states)] + beta_rates_array = [self.approximate_shape(E, Q_states[i], m_nu_squared, index[i]) * self.final_state_array[1][i] * approximate_e_phase_space for i in range(N_states)] @@ -696,7 +745,7 @@ def approximate_spectrum(self, E, Q, m_nu=0): index = np.where(E < Q-mnu) K = E[index] - nu_mass_shape = ((Q - K)**2 -m_nu)**0.5 + nu_mass_shape = ((Q - K)**2 -m_nu_squared)**0.5 beta_rates[index] = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.ephasespace(K, Q)*(Q - K)*nu_mass_shape return beta_rates @@ -928,9 +977,9 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No # energy resolution if self.resolution_model != 'two_gaussian': - lineshape = self.gauss_resolution_f(e_lineshape, 1, self.res, 0) + lineshape = self.gauss_resolution_f(e_lineshape, 1, self.res*self.width_scaling, 0) else: - lineshape = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_1, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_2, self.two_gaussian_mu_2) + lineshape = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_2*self.width_scaling, self.two_gaussian_mu_2) # spectrum shape spec = self.which_model(e_spec, endpoint, m_nu) @@ -953,33 +1002,10 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No # simplified lineshape - FWHM = 2.*np.sqrt(2.*np.log(2.))*self.res - - # sigmas = self.lineshape_p[1]+FWHM*self.lineshape_p[3] - # mus = self.lineshape_p[5]+self.lineshape_p[7]*np.log(FWHM-30) - # scatter_norm = 1 - - - - # if self.plot_lineshape: - # logger.info('Plotting lineshape') - # # start lineshape plot - # fig, ax = plt.subplots(1, 1, figsize=(7,5)) - # ax.plot(e_lineshape, lineshape, label='Unscattered resolution') - - - # for i in range(1, self.NScatters+1): - - - # gauss_i = self.gauss_resolution_f(e_lineshape, 1, sigmas[i-1], -mus[i-1]) - # lineshape += gauss_i*ratio**i - # scatter_norm += ratio**i - - # if self.plot_lineshape: - # ax.plot(e_lineshape, gauss_i*ratio**i, label='Scatter peak: {}'.format(i)) - - - # lineshape *= 1/scatter_norm + FWHM = 2.*np.sqrt(2.*np.log(2.))*self.res *self.width_scaling + if FWHM < 30.01: + #logger.warning('FWHM smaller 30. Setting to 30 instead.') + FWHM = 30.01 # get lineshape @@ -1022,7 +1048,7 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No if self.plot_lineshape: logger.info('Using Gaussian resolution model') else: - resolution = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_1, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_2, self.two_gaussian_mu_2) + resolution = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_2*self.width_scaling, self.two_gaussian_mu_2) if self.plot_lineshape: logger.info('Using two Gaussian resolution model') @@ -1031,7 +1057,10 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No plt.plot(e_lineshape, lineshape/np.max(lineshape), label = 'Full lineshape', color='Darkblue') - FWHM = 2.*np.sqrt(2.*np.log(2.))*self.res + FWHM = 2.*np.sqrt(2.*np.log(2.))*self.res*self.width_scaling + if FWHM < 30.01: + logger.warning('FWHM smaller 30. Setting to 30 instead.') + FWHM = 30.01 logger.info('Plotting lineshape for FWHM {}, probs {} and {} and hydrogen proportion {}.'.format(FWHM, prob_b, prob_c, self.hydrogen_proportion)) simple_ls, simple_norm = self.simplified_ls(e_lineshape, 0, FWHM, prob_b, prob_c) simple_ls = (self.gauss_resolution_f(e_lineshape, 1, self.res, 0)+simple_ls)/simple_norm @@ -1108,12 +1137,12 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No K_eff=K_convolved*efficiency # finally - K = K_eff/np.sum(K_eff)*np.sum(K_convolved) + K = K_eff/np.nansum(K_eff)*np.nansum(K_convolved) # if error from efficiency on spectrum shape should be returned if error: K_error=np.zeros((2, len(E))) - K_error = K_convolved*(efficiency_errors)/np.sum(K_eff)*np.sum(K_convolved) + K_error = K_convolved*(efficiency_errors)/np.nansum(K_eff)*np.nansum(K_convolved) return K, K_error else: @@ -1325,6 +1354,7 @@ def GenAndFit(params, counts, fit_config_dict,fit_options, sampled_priors, except Exception as e: print(e) + print(params) if event is not None: event.set() raise(e) @@ -1340,7 +1370,6 @@ def DoOneFit(data, fit_config_dict, fit_options, sampled_parameters={}, error_sc distorted = fit_options['distorted'] fit_nu_mass = fit_options['fit_nu_mass']""" - #print('do one fit', tilt, fit_tilt) T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) #T.is_scattered = scattered @@ -1372,7 +1401,7 @@ def DoOneFit(data, fit_config_dict, fit_options, sampled_parameters={}, error_sc def GetPDF(fit_config_dict, fit_options, params, plot=False): logger.info('Plotting lineshape: {}'.format(plot)) logger.info('PDF for params: {}'.format(params)) - logger.info(fit_options) + #logger.info(fit_options) #scattered = fit_options['scattered'] #distorted = fit_options['distorted'] From 6394d34f2130653197fea223e2ad189cc6fa035b Mon Sep 17 00:00:00 2001 From: cclaessens Date: Mon, 4 Oct 2021 15:31:45 -0700 Subject: [PATCH 123/199] added nuisance parameters --- .../processors/Fitters/BinnedDataFitter.py | 21 ++ .../Fitters/MCUncertaintyPropagation.py | 12 +- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 186 ++++++++++++------ 3 files changed, 158 insertions(+), 61 deletions(-) diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index edd2faac..9a1630cc 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -63,6 +63,7 @@ def InternalConfigure(self, params): self.constrained_parameters = reader.read_param(params, 'constrained_parameter_indices', []) self.constrained_means = reader.read_param(params, 'constrained_parameter_means', []) self.constrained_widths = reader.read_param(params, 'constrained_parameter_widths', []) + self.minos_cls = reader.read_param(params, 'minos_confidence_level_list', []) # derived configurations self.bin_centers = self.bins[0:-1]+0.5*(self.bins[1]-self.bins[0]) @@ -140,13 +141,33 @@ def fit(self): # minimze m_binned.simplex().migrad() m_binned.hesse() + #m_binned.minos() #self.param_states = m_binned.get_param_states() + #logger.info(self.param_states) self.m_binned = m_binned # results result_array = np.array(m_binned.values) error_array = np.array(m_binned.errors) + if len(self.minos_cls) > 0: + self.minos_errors = {} + for mcl in self.minos_cls: + logger.info('Getting minos errors for CL = {}'.format(mcl)) + try: + m_binned.minos(cl=mcl) + except RuntimeError as e: + print(m_binned.params) + raise e + self.minos_errors[mcl] = {} + if self.print_level: + logger.info(m_binned.merrors) + for k in m_binned.merrors.keys(): + self.minos_errors[mcl][k] = {'interval': [m_binned.merrors[k].lower, m_binned.merrors[k].upper], + 'number': m_binned.merrors[k].number, + 'name': m_binned.merrors[k].name, + 'is_valid': m_binned.merrors[k].is_valid} + if self.print_level == 1: logger.info('Fit results: {}'.format(result_array)) logger.info('Errors: {}'.format(error_array)) diff --git a/mermithid/processors/Fitters/MCUncertaintyPropagation.py b/mermithid/processors/Fitters/MCUncertaintyPropagation.py index ef72eeb8..78a0cd0d 100644 --- a/mermithid/processors/Fitters/MCUncertaintyPropagation.py +++ b/mermithid/processors/Fitters/MCUncertaintyPropagation.py @@ -9,6 +9,7 @@ from __future__ import absolute_import import numpy as np +from copy import deepcopy from morpho.utilities import morphologging, reader from morpho.processors import BaseProcessor @@ -38,7 +39,7 @@ def InternalConfigure(self, params): self.model = reader.read_param(params, 'model', "required") self.fit = reader.read_param(params, 'fit_function', "required") self.gen_and_fit = reader.read_param(params, 'gen_and_fit_function', "required") - self.fit_config_dict = reader.read_param(params, 'fit_config_dict', "required") + self.fit_config_dict = deepcopy(reader.read_param(params, 'fit_config_dict', "required")) self.fit_options = reader.read_param(params, 'fit_options', "optional") self.sample_parameters = reader.read_param(params, 'sample_parameters', []) self.stat_sys_combined = reader.read_param(params, 'stat_sys_combined', [True, True, True]) @@ -50,9 +51,13 @@ def InternalRun(self): self.results = {} + for k in self.fit_options.keys(): + self.fit_config_dict[k] = self.fit_options[k] + self.InitialFit() self.ParameterSampling() + #self.results['best_fit'] = list(self.fitted_params) return True @@ -74,8 +79,7 @@ def InitialFit(self): try: self.fitted_params, self.fitted_params_errors, self.Counts = self.fit(self.data, - self.fit_config_dict, - self.fit_options) + self.fit_config_dict) fit_successful = True except Exception as e: print(e) @@ -85,7 +89,6 @@ def InitialFit(self): logger.info('Best fit: {}'.format(self.fitted_params)) x, pdf, bins, fitted_model, asimov_data = self.model(self.fit_config_dict, - self.fit_options, params=self.fitted_params) return True @@ -126,7 +129,6 @@ def ParameterSampling(self): try: all_fit_returns.append(self.gen_and_fit(self.fitted_params, self.Counts, self.fit_config_dict, - self.fit_options, parameter_sampling[k_i], i, fixed_data)) fit_successful = True diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index b828603c..0163130a 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -67,14 +67,6 @@ def InternalConfigure(self, config_dict): self.fit_efficiency_tilt = reader.read_param(config_dict, 'fit_efficiency_tilt', False) # efficiency slope is free parameter self.use_asimov = False - self.fix_nu_mass = not self.fit_nu_mass - self.fix_endpoint = not reader.read_param(config_dict, 'fit_endpoint', True) - self.fix_background = not reader.read_param(config_dict, 'fit_background', True) - self.fix_amplitude = not reader.read_param(config_dict, 'fit_amplitude', True) - self.fix_tilt = not reader.read_param(config_dict, 'fit_tilt', False) - self.fix_scatter_ratio_b = not reader.read_param(config_dict, 'fit_scatter_peak_ratio_b', False) - self.fix_scatter_ratio_c = not reader.read_param(config_dict, 'fit_scatter_peak_ratio_c', False) - self.print_level = 0 # save plots in @@ -137,10 +129,10 @@ def InternalConfigure(self, config_dict): self.two_gaussian_mu_1 = reader.read_param(config_dict, 'two_gaussian_mu1', 0) self.two_gaussian_mu_2 = reader.read_param(config_dict, 'two_gaussian_mu2', 0) - self.two_gaussian_sig_1_mean = reader.read_param(config_dict, 'two_gaussian_sig1_mean', 15) - self.two_gaussian_sig_2_mean = reader.read_param(config_dict, 'two_gaussian_sig2_mean', 5) - self.two_gaussian_sig_1_width = reader.read_param(config_dict, 'two_gaussian_sig1_width', 1) - self.two_gaussian_sig_2_width = reader.read_param(config_dict, 'two_gaussian_sig2_width', 1) + self.two_gaussian_sigma_1_mean = reader.read_param(config_dict, 'two_gaussian_sigma_1_mean', 15) + self.two_gaussian_sigma_2_mean = reader.read_param(config_dict, 'two_gaussian_sigma_2_mean', 5) + self.two_gaussian_sigma_1_width = reader.read_param(config_dict, 'two_gaussian_sigma_1_width', 1) + self.two_gaussian_sigma_2_width = reader.read_param(config_dict, 'two_gaussian_sigma_2_width', 1) self.two_gaussian_wide_fraction = reader.read_param(config_dict, 'two_gaussian_wide_fraction', 1.) @@ -157,6 +149,64 @@ def InternalConfigure(self, config_dict): logger.warning('Some parameters are constrained: {} - {}'.format(self.constrained_parameters, self.constrained_parameter_names)) #self.print_level = 1 + + + self.fix_nu_mass = not self.fit_nu_mass + self.fix_endpoint = not reader.read_param(config_dict, 'fit_endpoint', True) + self.fix_background = not reader.read_param(config_dict, 'fit_background', True) + self.fix_amplitude = not reader.read_param(config_dict, 'fit_amplitude', True) + + self.nuisance_parameters = reader.read_param(config_dict, 'nuisance_parameters', {}) + if 'res' in self.nuisance_parameters.keys(): + self.fix_res = not self.nuisance_parameters['res'] + self.constrained_parameters.append(6) + self.constrained_means.append(self.res_mean) + self.constrained_widths.append(self.res_width) + else: + self.fix_res = True + + if 'scatter_peak_ratio_b' in self.nuisance_parameters.keys(): + self.fix_scatter_peak_ratio_b = not self.nuisance_parameters['scatter_peak_ratio_b'] + self.constrained_parameters.append(4) + self.constrained_means.append(self.scatter_peak_ratio_b_mean) + self.constrained_widths.append(self.scatter_peak_ratio_b_width) + else: + self.fix_scatter_peak_ratio_b = True + + if 'scatter_peak_ratio_c' in self.nuisance_parameters.keys(): + self.fix_scatter_peak_ratio_c = not self.nuisance_parameters['scatter_peak_ratio_c'] + self.constrained_parameters.append(5) + self.constrained_means.append(self.scatter_peak_ratio_c_mean) + self.constrained_widths.append(self.scatter_peak_ratio_c_width) + else: + self.fix_scatter_peak_ratio_c = True + + if 'two_gaussian_sigma_1' in self.nuisance_parameters.keys(): + self.fix_two_gaussian_sigma_1 = not self.nuisance_parameters['two_gaussian_sigma_1'] + self.constrained_parameters.append(7) + self.constrained_means.append(self.two_gaussian_sigma_1_mean) + self.constrained_widths.append(self.two_gaussian_sigma_1_width) + else: + self.fix_two_gaussian_sigma_1 = True + + if 'two_gaussian_sigma_2' in self.nuisance_parameters.keys(): + self.fix_two_gaussian_sigma_2 = not self.nuisance_parameters['two_gaussian_sigma_2'] + self.constrained_parameters.append(8) + self.constrained_means.append(self.two_gaussian_sigma_2_mean) + self.constrained_widths.append(self.two_gaussian_sigma_2_width) + else: + self.fix_two_gaussian_sigma_2 = True + + """if 'B' in self.nuisance_parameters.keys(): + self.fix_B = not self.nuisance_parameters['B'] + self.constrained_parameters.append(9) + self.constrained_means.append(self.B_mean) + self.constrained_widths.append(self.B_width)""" + + self.print_level = reader.read_param(config_dict, 'print_level', 0) + self.minos_intervals = reader.read_param(config_dict, 'minos_intervals', False) + self.minos_cls = [] + # frequency range self.min_frequency = reader.read_param(config_dict, 'min_frequency', "required") self.max_frequency = reader.read_param(config_dict, 'max_frequency', None) @@ -278,8 +328,8 @@ def InternalConfigure(self, config_dict): self.res = 30.01/float(2*np.sqrt(2*np.log(2))) self.endpoint=reader.read_param(config_dict, 'true_endpoint', 18.573e3) - self.two_gaussian_sig_1 = self.two_gaussian_sig_1_mean - self.two_gaussian_sig_2 = self.two_gaussian_sig_2_mean + self.two_gaussian_sigma_1 = self.two_gaussian_sigma_1_mean + self.two_gaussian_sigma_2 = self.two_gaussian_sigma_2_mean self.width_scaling = self.scale_mean #if self.use_fixed_scatter_peak_ratio: @@ -349,23 +399,36 @@ def InternalConfigure(self, config_dict): def ConfigureFit(self): # configure fit - energy_limits = [max([self.endpoint-1000, np.min(self.energies)+3.5]), min([self.endpoint+1000, np.max(self.energies)-3.5])] - #neutrino_limits = [-(np.max(self.energies) - energy_limits[1]-1)**2, (energy_limits[0]-np.min(self.energies)-1)**2] - neutrino_limits = [-300**2, 300**2] + + neutrino_limits = [-400**2, 400**2] + energy_limits = [max([self.endpoint-1000, np.min(self.energies)+np.sqrt(neutrino_limits[1])]), min([self.endpoint+1000, np.max(self.energies)-np.sqrt(np.abs(neutrino_limits[0]))])] #logger.warning('Neutrino mass limited to: {} - {}'.format(*np.sqrt(np.array(neutrino_limits)))) + if self.minos_intervals: + self.minos_cls = [0.683, 0.9] #logger.warning('Neutrino mass fitted: {}'.format(self.fit_nu_mass)) if not self.fit_efficiency_tilt: - self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', 'scatter_peak_ratio_b', 'scatter_peak_ratio_c'] - self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c] - self.fixes = [self.fix_endpoint, self.fix_background, self.fix_nu_mass, self.fix_amplitude, self.fix_scatter_ratio_b, self.fix_scatter_ratio_c] + self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', + 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', + 'res', 'two_gaussia_sigma_1', 'two_gaussian_sigma_2'] + self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, + self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, + self.res, self.two_gaussian_sigma_1, self.two_gaussian_sigma_2] + + self.fixes = [self.fix_endpoint, self.fix_background, self.fix_nu_mass, self.fix_amplitude, + self.fix_scatter_peak_ratio_b, self.fix_scatter_peak_ratio_c, + self.fix_res, self.fix_two_gaussian_sigma_1, self.fix_two_gaussian_sigma_2] + self.limits = [energy_limits, - [1e-10, None], - neutrino_limits, [0, None], + neutrino_limits, + [100, None], + [0.1, 1.], [0.1, 1.], - [0.1, 1.]] + [30, 100], + [1, 100], + [1, 100]] @@ -393,10 +456,10 @@ def ConfigureFit(self): logger.info('Fixed: {}'.format(self.fixes)) logger.info('Initial values: {}'.format(self.initial_values)) - if len(self.constrained_parameters) > 0: + #if len(self.constrained_parameters) > 0: #logger.info('{}\n{}'.format(self.constrained_parameters, self.parameter_names)) #self.constrained_parameters = [self.parameter_names.index(p) for p in self.constrained_parameter_names[0]] - self.print_level=0 + #self.print_level=0 @@ -575,6 +638,11 @@ def ConvertAndHistogram(self, weights=None): def SamplePriors(self, sampled_parameters): + + for k in sampled_parameters.keys(): + if k in self.nuisance_parameters and self.nuisance_parameters[k]: + raise ValueError('{} is nuisance parameter.'.format(k)) + logger.info('Sampling: {}'.format([k for k in sampled_parameters.keys() if sampled_parameters[k]])) self.parameter_samples = {} sample_values = [] @@ -585,14 +653,14 @@ def SamplePriors(self, sampled_parameters): self.res = 30.01/float(2*np.sqrt(2*np.log(2))) self.parameter_samples['res'] = self.res sample_values.append(self.res) - if 'two_gaussian_std_1' in sampled_parameters.keys() and sampled_parameters['two_gaussian_std_1']: - self.two_gaussian_sig_1 = self.Gaussian_sample(self.two_gaussian_sig_1_mean, self.two_gaussian_sig_1_width) - self.parameter_samples['two_gaussian_std_1'] = self.two_gaussian_sig_1 - sample_values.append(self.two_gaussian_sig_1) - if 'two_gaussian_std_2' in sampled_parameters.keys() and sampled_parameters['two_gaussian_std_2']: - self.two_gaussian_sig_2 = self.Gaussian_sample(self.two_gaussian_sig_2_mean, self.two_gaussian_sig_2_width) - self.parameter_samples['two_gaussian_std_2'] = self.two_gaussian_sig_2 - sample_values.append(self.two_gaussian_sig_2) + if 'two_gaussian_sigma_1' in sampled_parameters.keys() and sampled_parameters['two_gaussian_sigma_1']: + self.two_gaussian_sigma_1 = self.Gaussian_sample(self.two_gaussian_sigma_1_mean, self.two_gaussian_sigma_1_width) + self.parameter_samples['two_gaussian_sigma_1'] = self.two_gaussian_sigma_1 + sample_values.append(self.two_gaussian_sigma_1) + if 'two_gaussian_sigma_2' in sampled_parameters.keys() and sampled_parameters['two_gaussian_sigma_2']: + self.two_gaussian_sigma_2 = self.Gaussian_sample(self.two_gaussian_sigma_2_mean, self.two_gaussian_sigma_2_width) + self.parameter_samples['two_gaussian_sigma_2'] = self.two_gaussian_sigma_2 + sample_values.append(self.two_gaussian_sigma_2) if 'scatter_peak_ratio' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio']: self.scatter_peak_ratio_b = self.Beta_sample(self.scatter_peak_ratio_mean, self.scatter_peak_ratio_width) self.scatter_peak_ratio_c = 1 @@ -940,7 +1008,7 @@ def running_mean(self, x, N): return (cumsum[int(N_round)::int(N_round)] - cumsum[:-int(N_round):int(N_round)]) / float(N_round) - def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=None, tilt=0., error=False): + def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=None, res=None, sig1=None, sig2=None, tilt=0., error=False): E = np.array(E) if len(E)==0: @@ -977,9 +1045,9 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No # energy resolution if self.resolution_model != 'two_gaussian': - lineshape = self.gauss_resolution_f(e_lineshape, 1, self.res*self.width_scaling, 0) + lineshape = self.gauss_resolution_f(e_lineshape, 1, res*self.width_scaling, 0) else: - lineshape = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_2*self.width_scaling, self.two_gaussian_mu_2) + lineshape = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, two_gaussian_sigma_1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, two_gaussian_sigma_2*self.width_scaling, self.two_gaussian_mu_2) # spectrum shape spec = self.which_model(e_spec, endpoint, m_nu) @@ -1002,7 +1070,7 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No # simplified lineshape - FWHM = 2.*np.sqrt(2.*np.log(2.))*self.res *self.width_scaling + FWHM = 2.*np.sqrt(2.*np.log(2.))*res *self.width_scaling if FWHM < 30.01: #logger.warning('FWHM smaller 30. Setting to 30 instead.') FWHM = 30.01 @@ -1044,11 +1112,11 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No plt.figure(figsize=(7,5)) #plt.plot(e_lineshape, self.simplified_ls(e_lineshape, 0, FWHM, ratio), color='red', label='FDG') if self.resolution_model != 'two_gaussian': - resolution = self.gauss_resolution_f(e_lineshape, 1, self.res, 0) + resolution = self.gauss_resolution_f(e_lineshape, 1, res, 0) if self.plot_lineshape: logger.info('Using Gaussian resolution model') else: - resolution = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sig_2*self.width_scaling, self.two_gaussian_mu_2) + resolution = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sigma_1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sigma_2*self.width_scaling, self.two_gaussian_mu_2) if self.plot_lineshape: logger.info('Using two Gaussian resolution model') @@ -1057,13 +1125,13 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No plt.plot(e_lineshape, lineshape/np.max(lineshape), label = 'Full lineshape', color='Darkblue') - FWHM = 2.*np.sqrt(2.*np.log(2.))*self.res*self.width_scaling + FWHM = 2.*np.sqrt(2.*np.log(2.))*res*self.width_scaling if FWHM < 30.01: logger.warning('FWHM smaller 30. Setting to 30 instead.') FWHM = 30.01 logger.info('Plotting lineshape for FWHM {}, probs {} and {} and hydrogen proportion {}.'.format(FWHM, prob_b, prob_c, self.hydrogen_proportion)) simple_ls, simple_norm = self.simplified_ls(e_lineshape, 0, FWHM, prob_b, prob_c) - simple_ls = (self.gauss_resolution_f(e_lineshape, 1, self.res, 0)+simple_ls)/simple_norm + simple_ls = (self.gauss_resolution_f(e_lineshape, 1, res, 0)+simple_ls)/simple_norm plt.plot(e_lineshape, simple_ls/np.nanmax(simple_ls), label='Hydrogen only lineshape', color='red') plt.xlabel('Energy [eV]') plt.ylabel('Amplitude') @@ -1150,18 +1218,18 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No - def TritiumSpectrumBackground(self, E=[], endpoint=18.6e3, background=0, m_nu=0, amplitude=1., prob_b=None, prob_c=None, tilt=0., error=False): + def TritiumSpectrumBackground(self, E=[], endpoint=18.6e3, background=0, m_nu=0, amplitude=1., prob_b=None, prob_c=None, res=None, sig1=None, sig2=None, tilt=0., error=False): if len(E)==0: E = self.bin_centers if error: - K, K_error = self.TritiumSpectrum(E, endpoint, m_nu, prob_b, prob_c, tilt, error) - K_norm = np.sum(self.TritiumSpectrum(self._energies, endpoint, m_nu, prob_b, prob_c, tilt, error=False)*(self._energies[1]-self._energies[0])) + K, K_error = self.TritiumSpectrum(E, endpoint, m_nu, prob_b, prob_c, res, sig1, sig2, tilt, error) + K_norm = np.sum(self.TritiumSpectrum(self._energies, endpoint, m_nu, prob_b, prob_c, res, sig1, sig2, tilt, error=False)*(self._energies[1]-self._energies[0])) else: - K = self.TritiumSpectrum(E, endpoint, m_nu, prob_b, prob_c, tilt, error) - K_norm = np.sum(self.TritiumSpectrum(self._energies, endpoint, m_nu, prob_b, prob_c, tilt, error=False)*(self._energies[1]-self._energies[0])) + K = self.TritiumSpectrum(E, endpoint, m_nu, prob_b, prob_c, res, sig1, sig2, tilt, error) + K_norm = np.sum(self.TritiumSpectrum(self._energies, endpoint, m_nu, prob_b, prob_c, res, sig1, sig2, tilt, error=False)*(self._energies[1]-self._energies[0])) @@ -1193,14 +1261,14 @@ def TritiumSpectrumBackground(self, E=[], endpoint=18.6e3, background=0, m_nu=0, else: return K+B - def normalized_TritiumSpectrumBackground(self, E=[], endpoint=18.6e3, background=0, m_nu=0., amplitude=1., prob_b=None, prob_c=None, tilt=0., error=False): + def normalized_TritiumSpectrumBackground(self, E=[], endpoint=18.6e3, background=0, m_nu=0., amplitude=1., prob_b=None, prob_c=None, res=None, sig1=None, sig2=None, tilt=0., error=False): if error: - t, t_error = self.TritiumSpectrumBackground(E, endpoint, background, m_nu, amplitude, prob_b, prob_c, tilt, error=error) + t, t_error = self.TritiumSpectrumBackground(E, endpoint, background, m_nu, amplitude, prob_b, prob_c, res, sig1, sig2, tilt, error=error) t_norm = np.sum(t) return t/t_norm, t_error/t_norm else: - t = self.TritiumSpectrumBackground(E, endpoint, background, m_nu, amplitude, prob_b, prob_c, tilt, error=error) + t = self.TritiumSpectrumBackground(E, endpoint, background, m_nu, amplitude, prob_b, prob_c, res, sig1, sig2, tilt, error=error) t_norm = np.sum(t) return t/t_norm @@ -1298,8 +1366,8 @@ def Efficiency(self, E, freq=False, pseudo=False): # Functions using model above -def GenAndFit(params, counts, fit_config_dict,fit_options, sampled_priors, - i, fixed_data = [], error_scaling=1, tilt = None, fit_tilt = False, event=None): +def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, + i=0, fixed_data = [], error_scaling=1, tilt = None, fit_tilt = False, event=None): """if 'scattered' in fit_options.keys(): scattered = fit_options['scattered'] @@ -1360,10 +1428,13 @@ def GenAndFit(params, counts, fit_config_dict,fit_options, sampled_priors, raise(e) else: - return results, errors, parameter_samples + if fit_config_dict['minos_intervals']: + return results, T.minos_errors, parameter_samples + else: + return results, errors, parameter_samples -def DoOneFit(data, fit_config_dict, fit_options, sampled_parameters={}, error_scaling=0, +def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, tilt=None, fit_tilt = False, data_is_energy=False): """scattered = fit_options['scattered'] @@ -1372,6 +1443,7 @@ def DoOneFit(data, fit_config_dict, fit_options, sampled_parameters={}, error_sc T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) + T.print_level=1 #T.is_scattered = scattered #T.is_distorted = distorted T.error_scaling = error_scaling @@ -1395,10 +1467,12 @@ def DoOneFit(data, fit_config_dict, fit_options, sampled_parameters={}, error_sc results, errors = T.SampleConvertAndFit(sampled_parameters) total_counts = results[1]+results[3] + if fit_config_dict['minos_intervals']: + return results, T.minos_errors, total_counts + else: + return results, errors, total_counts - return results, errors, total_counts - -def GetPDF(fit_config_dict, fit_options, params, plot=False): +def GetPDF(fit_config_dict, params, plot=False): logger.info('Plotting lineshape: {}'.format(plot)) logger.info('PDF for params: {}'.format(params)) #logger.info(fit_options) From 10f24d59b87f5a333d5f6f847be6efe91350ff79 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Thu, 7 Oct 2021 18:19:11 -0400 Subject: [PATCH 124/199] Testing channel runtime correction --- mermithid/misc/FakeTritiumDataFunctions.py | 21 ++++---- .../TritiumSpectrum/FakeDataGenerator.py | 54 ++++++++++++++++--- 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 6a39f766..7d53fa44 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -316,19 +316,20 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) #Convolving - if (lineshape=='detailed_scattering' or lineshape=='detailed'): - if (resolution_function == 'simulated_resolution' or resolution_function == 'simulated'): - convolved_list = [] - for j in range(len(lineshape_rates)): - beta_rates = spectral_rate(K_segments[j], Q, mnu, final_state_array) - convolved_list.append(convolve(beta_rates, lineshape_rates[j], mode='same')) - convolved = np.concatenate(convolved_list, axis=None) - else: - lineshape_rates = np.flipud(lineshape_rates) - else: + if (lineshape=='detailed_scattering' or lineshape=='detailed') and (resolution_function == 'simulated_resolution' or resolution_function == 'simulated'): + convolved_list = [] + for j in range(len(lineshape_rates)): + beta_rates = spectral_rate(K_segments[j], Q, mnu, final_state_array) + convolved_list.append(convolve(beta_rates, lineshape_rates[j], mode='same')) + convolved = np.concatenate(convolved_list, axis=None) + elif resolution_function=='gaussian': + lineshape_rates = np.flipud(lineshape_rates) beta_rates = spectral_rate(K, Q, mnu, final_state_array) convolved = convolve(beta_rates, lineshape_rates, mode='same') + if (lineshape=='gaussian' or lineshape=='simplified_scattering' or lineshape=='simplified'): + beta_rates = spectral_rate(K, Q, mnu, final_state_array) + convolved = convolve(beta_rates, lineshape_rates, mode='same') below_Kmin = np.where(K < Kmin) np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 364803ec..6f67b6c8 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -46,9 +46,9 @@ def InternalConfigure(self, params): - B_field: used for energy-frequency conversion - sig_trans [eV]: width of thermal broadening - other_sig [eV]: width of other broadening - - runtime [s]: used to calculate number of background events + - channel_runtimes [s]: live time for each channel + - channel_bounds [Hz]: inner bounds between channels (one less than the number of channels) - S: number of signal events - - A_b [1/eV/s]: background rate - poisson_stats (boolean): if True number of total events is random - err_from_B [eV]: energy uncertainty originating from B uncertainty – gases: list of strings naming gases to be included in complex lineshape model. Options: 'H2', 'He', 'Kr', 'Ar', 'CO' @@ -112,11 +112,12 @@ def InternalConfigure(self, params): self.broadening = np.sqrt(self.sig_trans**2+self.other_sig**2) #Total energy broadening (eV) # Phase II Spectrum parameters - self.runtime = reader.read_param(params, 'runtime', 6.57e6) #In seconds. Default time is ~2.5 months. + self.channel_runtimes = reader.read_param(params, 'channel_runtimes', [7185228., 7129663., 7160533.]) + self.channel_bounds = reader.read_param(params, 'channel_bounds', [1.38623121e9+24.5e9, 1.44560621e9+24.5e9]) self.S = reader.read_param(params, 'S', 3300) self.B_1kev = reader.read_param(params, 'B_1keV', 0.1) #Background rate per keV for full runtime - self.A_b = reader.read_param(params, 'A_b', self.B_1kev/float(self.runtime)/1000.) #Flat background activity: events/s/eV - self.B =self.A_b*self.runtime*(self.Kmax-self.Kmin) #Background poisson rate + #self.A_b = reader.read_param(params, 'A_b', self.B_1kev/float(self.runtime)/1000.) #Flat background activity: events/s/eV #No longer in use + #self.B =self.A_b*self.runtime*(self.Kmax-self.Kmin) #Background poisson rate #No longer in use self.poisson_stats = reader.read_param(params, 'poisson_stats', True) self.err_from_B = reader.read_param(params, 'err_from_B', 0.) #In eV, kinetic energy error from f_c --> K conversion @@ -422,21 +423,60 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 ratesS = convolve(ratesS, gaussian_rates, mode='same') ratesB = convolve(ratesB, gaussian_rates, mode='same') - ratesS[ratesS<0.] = 0. ratesB[ratesB<0.] = 0. rate_sumS, rate_sumB = np.sum(ratesS), np.sum(ratesB) probsS = np.array(ratesS)/rate_sumS probsB = np.array(ratesB)/rate_sumB - self.probs = (S*probsS + B*probsB)/(S+B) + + #Calculate three different probs variables, for each of the three runtimes + runtime_ratios = [t/float(self.channel_runtimes[0]) for t in self.channel_runtimes] logger.info('Generating data') time4 = time.time() + #Break up self.Koptions into three different arrays. + #Then, sample KE variables for each of the arrays and appropriate elements of self.channel_runtimes and self.probs. + #Finally, concatenate together the three KE arrays. + temp_Koptions, temp_probsS, temp_probsB = self.Koptions, probsS, probsB + split_Koptions, split_probsS, split_probsB = [], [], [] + for i in range(len(self.channel_bounds)): + print(len(temp_Koptions), len(temp_probsS), len(temp_probsB)) + split_Koptions.append(temp_Koptions[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) + split_probsS.append(temp_probsS[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) + split_probsB.append(temp_probsB[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) + temp_probsS = temp_probsS[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] + temp_probsB = temp_probsB[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] + temp_Koptions = temp_Koptions[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] + + split_Koptions.append(temp_Koptions) + split_probsS.append(temp_probsS) + split_probsB.append(temp_probsB) + + self.probs = [] + for i in range(len(self.channel_runtimes)): + self.probs.append((S*runtime_ratios[i]*split_probsS[i] + B*split_probsB[i])/(S*runtime_ratios[i]+B)) + + print(len(split_Koptions[0])) + print(len(self.probs[0])) + + self.Koptions = np.concatenate(split_Koptions) + self.probs = np.concatenate(self.probs) + if self.poisson_stats: KE = np.random.choice(self.Koptions, np.random.poisson(S+B), p = self.probs) else: KE = np.random.choice(self.Koptions, round(S+B), p = self.probs) + + """ + split_KE = [] + for i in range(len(runtime_ratios)): + if self.poisson_stats: + split_KE.append(np.random.choice(split_Koptions[i], np.random.poisson(S*runtime_ratios[i]+B), p = self.probs[i])) + else: + split_KE.append(np.random.choice(split_Koptions[i], round(S*runtime_ratios[i]+B), p = self.probs[i])) + """ + time5 = time.time() logger.info('... took {} s'.format(time5-time4)) From cb59f8609740bf952ac6fa55122758353cfa3b00 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 13 Oct 2021 16:43:06 -0700 Subject: [PATCH 125/199] commenting and re-organizing tritium frequentist analysis --- mermithid/misc/FakeTritiumDataFunctions.py | 1 + .../processors/Fitters/BinnedDataFitter.py | 8 +- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 695 +++++++++--------- .../TritiumSpectrum/FakeDataGenerator.py | 1 + 4 files changed, 362 insertions(+), 343 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index e96bdbdd..b8c219e2 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -282,6 +282,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, """K is an array-like object """ logger.info('Using scipy convolve') + logger.info('Lineshape is {} with {}'.format(lineshape, resolution_function)) energy_half_range = max(max_energy, abs(min_energy)) if ins_res_width_bounds != None: diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index 9a1630cc..046b96f7 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -64,6 +64,7 @@ def InternalConfigure(self, params): self.constrained_means = reader.read_param(params, 'constrained_parameter_means', []) self.constrained_widths = reader.read_param(params, 'constrained_parameter_widths', []) self.minos_cls = reader.read_param(params, 'minos_confidence_level_list', []) + self.minos_intervals = reader.read_param(params,'find_minos_intervals', False) # derived configurations self.bin_centers = self.bins[0:-1]+0.5*(self.bins[1]-self.bins[0]) @@ -121,12 +122,7 @@ def fit(self): m_binned = Minuit(self.negPoissonLogLikelihood, self.initial_values, - # error=self.parameter_errors, - # errordef = 0.5, limit = self.limits, name=self.parameter_names, - # fix=self.fixes, - # print_level=self.print_level, - # throw_nan=True ) m_binned.errordef = 0.5 @@ -150,7 +146,7 @@ def fit(self): result_array = np.array(m_binned.values) error_array = np.array(m_binned.errors) - if len(self.minos_cls) > 0: + if self.minos_intervals: self.minos_errors = {} for mcl in self.minos_cls: logger.info('Getting minos errors for CL = {}'.format(mcl)) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 0163130a..58e9620e 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -31,31 +31,159 @@ from morpho.utilities import morphologging, reader logger = morphologging.getLogger(__name__) -#import helper_functions.plot_methods as pm -from mermithid.misc.DetectionEfficiencyUtilities import * -#import importlib -#importlib.reload(det_eff) -electron_mass = constants.electron_mass/constants.e*constants.c**2 -FineStructureConstant = 0.0072973525664 +from mermithid.misc.DetectionEfficiencyUtilities import pseudo_integrated_efficiency, integrated_efficiency -# # for converting numpy array to double -# def float2double(a): -# """ -# convert floats (or array of floats) to double -# """ -# if a is None or a.dtype == np.float64: -# return a -# else: -# return a.astype(np.float64) +#electron_mass = constants.electron_mass/constants.e*constants.c**2 +#FineStructureConstant = 0.0072973525664 + +############################################################################### +# Functions that can be imported from here and facitilate working with the processor +# They create an instance of the processor defined below and use it for analysis +# GenAndFit tells the processor to generate new fake data and fit it +# DoOneFit give the processor data and tells it to fit it +# GetPDF returns model pdf for some parameters + + +def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, + i=0, fixed_data = [], error_scaling=1, tilt = None, + fit_tilt = False, event=None): + + """if 'scattered' in fit_options.keys(): + scattered = fit_options['scattered'] + else: + scattered = None + if 'distorted' in fit_options.keys(): + distorted = fit_options['distorted'] + else: + distorted = None + if 'fit_nu_mass' in fit_options.keys(): + fit_nu_mass = fit_options['fit_nu_mass'] + else: + fit_nu_mass = None""" + + if i%20 == 0: + logger.info('Sampling: {}'.format(i)) + + + + try: + + T = BinnedTritiumMLFitter("TritiumFitter") + T.InternalConfigure(fit_config_dict) + #T.is_scattered = scattered + #T.is_distorted = distorted + T.error_scaling = error_scaling + T.integrate_bins = True + + if tilt is not None:# and tilt !=0: + T.tilted_efficiency = True + T.tilt = tilt + + + + # generate random data from best fit parameters + if len(fixed_data) == 0: + _, new_data = T.GenerateData(params, counts) + + else: + _, new_data = T.GenerateAsimovData(params) + + + #if fit_nu_mass: + # T.fix_nu_mass = False + + if fit_tilt: + T.fix_tilt = False + + T.freq_data = new_data + results, errors = T.SampleConvertAndFit(sampled_priors, params) + parameter_samples = T.parameter_samples + + except Exception as e: + print(e) + print(params) + if event is not None: + event.set() + raise(e) + + else: + if fit_config_dict['minos_intervals']: + return results, T.minos_errors, parameter_samples + else: + return results, errors, parameter_samples + + +def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, + tilt=None, fit_tilt = False, data_is_energy=False): + + """scattered = fit_options['scattered'] + distorted = fit_options['distorted'] + fit_nu_mass = fit_options['fit_nu_mass']""" + + T = BinnedTritiumMLFitter("TritiumFitter") + T.InternalConfigure(fit_config_dict) + T.print_level=1 + #T.is_scattered = scattered + #T.is_distorted = distorted + T.error_scaling = error_scaling + T.integrate_bins = True + logger.info('Energy stepsize: {}'.format(T.denergy)) + + if tilt is not None:# and tilt != 0: + T.tilted_efficiency = True + T.tilt = tilt + + #if fit_nu_mass: + # logger.info('Fitting neutrino mass') + # T.fix_nu_mass = False + if fit_tilt: + logger.info('Going to fit efficiency tilt') + T.fix_tilt = False + + if data_is_energy: + data = T.Frequency(data) + T.freq_data = data + results, errors = T.SampleConvertAndFit(sampled_parameters) + total_counts = results[1]+results[3] + + if fit_config_dict['minos_intervals']: + return results, T.minos_errors, total_counts + else: + return results, errors, total_counts + +def GetPDF(fit_config_dict, params, plot=False): + logger.info('Plotting lineshape: {}'.format(plot)) + logger.info('PDF for params: {}'.format(params)) + #logger.info(fit_options) + #scattered = fit_options['scattered'] + #distorted = fit_options['distorted'] + + T = BinnedTritiumMLFitter("TritiumFitter") + T.InternalConfigure(fit_config_dict) + T.plot_lineshape = plot + #T.is_scatterd=scattered + #T.is_distorted=distorted + + pdf = T.TritiumSpectrumBackground(T.energies, *params) + _, asimov_binned_data = T.GenerateAsimovData(params) + binned_fit = T.TritiumSpectrumBackground(T.bin_centers, *params, error=True) + + return T.energies, pdf, T.bin_centers, binned_fit, asimov_binned_data + + +############################################################################## +# Processor definition class BinnedTritiumMLFitter(BinnedDataFitter): def InternalConfigure(self, config_dict): - # model options + # ==================================== + # Model options and parameter settings + # ==================================== self.use_approx_model = reader.read_param(config_dict, 'use_approximate_model', True) self.use_toy_model_efficiency = reader.read_param(config_dict, 'use_toy_model_efficiency', False) self.fit_nu_mass = reader.read_param(config_dict, 'fit_neutrino_mass', False) @@ -66,10 +194,15 @@ def InternalConfigure(self, config_dict): self.integrate_bins = reader.read_param(config_dict, 'integrate_bins', True) # integrate spectrum over bin widths self.fit_efficiency_tilt = reader.read_param(config_dict, 'fit_efficiency_tilt', False) # efficiency slope is free parameter + + # final state spectrum + self.use_final_states = reader.read_param(config_dict, 'use_final_states', False) + self.final_state_array = reader.read_param(config_dict, 'final_states_array', [[0], [1]]) + self.use_asimov = False - # save plots in + # save plots in (processor can plot lineshape used in tritium model) self.savepath = reader.read_param(config_dict, 'savepath', '.') @@ -79,16 +212,13 @@ def InternalConfigure(self, config_dict): self.endpoint_mean = reader.read_param(config_dict,'endpoint_mean', 18.6e3) self.endpoint_width = reader.read_param(config_dict, 'endpoint_width', 1) - # efficiency - self.efficiency_file_path = reader.read_param(config_dict, 'efficiency_file_path', '') - - - # final state spectrum - self.use_final_states = reader.read_param(config_dict, 'use_final_states', False) - self.final_state_array = reader.read_param(config_dict, 'final_states_array', [[0], [1]]) + # frequency range + self.min_frequency = reader.read_param(config_dict, 'min_frequency', "required") + self.max_frequency = reader.read_param(config_dict, 'max_frequency', None) - #logger.info('final state array: {}'.format(self.final_state_array)) + # path to json with efficiency dictionary + self.efficiency_file_path = reader.read_param(config_dict, 'efficiency_file_path', '') # detector response self.NScatters = reader.read_param(config_dict, 'NScatters', 20) @@ -136,26 +266,26 @@ def InternalConfigure(self, config_dict): self.two_gaussian_wide_fraction = reader.read_param(config_dict, 'two_gaussian_wide_fraction', 1.) - ######################### + # ================= # fit configuration - ######################## + # ================= + + self.print_level = reader.read_param(config_dict, 'print_level', 0) + self.fix_nu_mass = not self.fit_nu_mass + self.fix_endpoint = not reader.read_param(config_dict, 'fit_endpoint', True) + self.fix_background = not reader.read_param(config_dict, 'fit_background', True) + self.fix_amplitude = not reader.read_param(config_dict, 'fit_amplitude', True) self.counts_guess = reader.read_param(config_dict, 'counts_guess', 5000) self.mass_guess = reader.read_param(config_dict, 'nu_mass_guess', 0.0) + + + # Parameters can be constrained manually or by inlcuding them in the nuisance parameter dictionary self.constrained_parameter_names = reader.read_param(config_dict, 'constrained_parameter_names', []) self.constrained_parameters = reader.read_param(config_dict, 'constrained_parameters', []) self.constrained_means = reader.read_param(config_dict, 'constrained_means', []) self.constrained_widths = reader.read_param(config_dict, 'constrained_widths', []) - if len(self.constrained_parameters) > 0: - logger.warning('Some parameters are constrained: {} - {}'.format(self.constrained_parameters, self.constrained_parameter_names)) - #self.print_level = 1 - - self.fix_nu_mass = not self.fit_nu_mass - self.fix_endpoint = not reader.read_param(config_dict, 'fit_endpoint', True) - self.fix_background = not reader.read_param(config_dict, 'fit_background', True) - self.fix_amplitude = not reader.read_param(config_dict, 'fit_amplitude', True) - self.nuisance_parameters = reader.read_param(config_dict, 'nuisance_parameters', {}) if 'res' in self.nuisance_parameters.keys(): self.fix_res = not self.nuisance_parameters['res'] @@ -203,20 +333,22 @@ def InternalConfigure(self, config_dict): self.constrained_means.append(self.B_mean) self.constrained_widths.append(self.B_width)""" - self.print_level = reader.read_param(config_dict, 'print_level', 0) - self.minos_intervals = reader.read_param(config_dict, 'minos_intervals', False) - self.minos_cls = [] - - # frequency range - self.min_frequency = reader.read_param(config_dict, 'min_frequency', "required") - self.max_frequency = reader.read_param(config_dict, 'max_frequency', None) + if len(self.constrained_parameters) > 0: + logger.warning('Some parameters are constrained: {} - {}'.format(self.constrained_parameters, self.constrained_parameter_names)) + #self.print_level = 1 + # MC uncertainty propagation does not need the fit uncertainties returned by iminuit. uncertainties are instead obtained from the distribution of fit results. + # But if the uncertainty is unstead propagated by adding constraiend nuisance parameters then the fit uncertainties are needed. + # imnuit can calculated hesse and minos intervals. the former are symmetric. we want the asymetric intrevals-> minos + # minos_cls is the list of uncertainty level that should be obtained: e.g. [0.68, 0.9] + self.minos_intervals = reader.read_param(config_dict, 'minos_intervals', False) + self.minos_cls = reader.read_param(config_dict, 'minos_confidence_levels', [0.683, 0.9]) - ######################################### - # detection efficiency specification - ######################################### + # ================================== + # detection efficiency details + # ================================== self.tilted_efficiency = False self.tilt = 0. @@ -250,14 +382,14 @@ def InternalConfigure(self, config_dict): if self.max_frequency == None: raise ValueError('Max frequency undetermined') - ######################################### - # lineshape specification - ######################################### + # ================================== + # lineshape details + # ================================== # simplified lineshape parameters self.lineshape_p = np.loadtxt(self.simplified_lineshape_path, unpack=True) - # if true lineshape is plotted during tritium spectrum shape generation + # if true lineshape is plotted during tritium spectrum shape generation (for debugging) self.plot_lineshape = False # helium lineshape @@ -317,9 +449,9 @@ def InternalConfigure(self, config_dict): - ######################################### + # ================================== # initial values - ######################################### + # ================================== self.B = self.B_mean self.res = self.res_mean @@ -328,6 +460,7 @@ def InternalConfigure(self, config_dict): self.res = 30.01/float(2*np.sqrt(2*np.log(2))) self.endpoint=reader.read_param(config_dict, 'true_endpoint', 18.573e3) + self.background=reader.read_param(config_dict, 'initial_background', 0) self.two_gaussian_sigma_1 = self.two_gaussian_sigma_1_mean self.two_gaussian_sigma_2 = self.two_gaussian_sigma_2_mean self.width_scaling = self.scale_mean @@ -341,11 +474,14 @@ def InternalConfigure(self, config_dict): self.parameter_samples = {} - ######################################### + # ================================== # energies and bins - ######################################### + # ================================== - # internal variables + # bin width used for poisson statistics fit + # energy stepsize is used for integration approximation: + # counts in a bin are integrated spectrum over bin width. + # integration is approximated by summing over a number of discrete steps in a bin. self.dbins = reader.read_param(config_dict, 'energy_bin_width', 50) self.denergy = reader.read_param(config_dict, 'energy_step_size', min([np.round(self.dbins/10, 2), 1])) @@ -364,109 +500,99 @@ def InternalConfigure(self, config_dict): if len(self._bins) > self.N_bins: self._bins = self._bins[:-1] - #print(len(self._energies), self.N_energy_bins) - #print(len(self._bins), self.N_bins) - #print(self.dbins/self.denergy) - - self.bin_centers = self._bins[0:-1]+0.5*(self._bins[1]-self._bins[0]) self.freq_bins = self.Frequency(self._bins) self.freq_bin_centers = self.Frequency(self.bin_centers) - #self._hist = [] - #self._data = [] - + # arrays for internal usage self._bin_efficiency, self._bin_efficiency_errors = [], [] self._full_efficiency, self._full_efficiency_errors = [], [] self._bin_efficiency, self._bin_efficiency_errors = self.Efficiency(self.bin_centers) self._full_efficiency, self._full_efficiency_errors = self.Efficiency(self.energies) - ######################################### + # ================================== # configure parent BinnedDataFitter - ######################################### + # ================================== - # overwrite model + # This processor inherits from the BinnedDataFitter that does binned max likelihood fitting + # overwrite parent model with tritium model used here self.model = self.TritiumSpectrumBackground # now configure fit self.ConfigureFit() - return True + def ConfigureFit(self): # configure fit + # this should not be hard coded neutrino_limits = [-400**2, 400**2] energy_limits = [max([self.endpoint-1000, np.min(self.energies)+np.sqrt(neutrino_limits[1])]), min([self.endpoint+1000, np.max(self.energies)-np.sqrt(np.abs(neutrino_limits[0]))])] - #logger.warning('Neutrino mass limited to: {} - {}'.format(*np.sqrt(np.array(neutrino_limits)))) + if self.print_level == 1: + logger.warning('Neutrino mass limited to: {} - {}'.format(*np.sqrt(np.array(neutrino_limits)))) - if self.minos_intervals: - self.minos_cls = [0.683, 0.9] #logger.warning('Neutrino mass fitted: {}'.format(self.fit_nu_mass)) - if not self.fit_efficiency_tilt: - self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', - 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', - 'res', 'two_gaussia_sigma_1', 'two_gaussian_sigma_2'] - self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, - self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, - self.res, self.two_gaussian_sigma_1, self.two_gaussian_sigma_2] - - self.fixes = [self.fix_endpoint, self.fix_background, self.fix_nu_mass, self.fix_amplitude, - self.fix_scatter_peak_ratio_b, self.fix_scatter_peak_ratio_c, - self.fix_res, self.fix_two_gaussian_sigma_1, self.fix_two_gaussian_sigma_2] - - self.limits = [energy_limits, - [0, None], - neutrino_limits, - [100, None], - [0.1, 1.], - [0.1, 1.], - [30, 100], - [1, 100], - [1, 100]] - - - - self.parameter_errors = [max([0.1, 0.1*p]) for p in self.initial_values] - - - else: - logger.warning('Efficiency tilt will be fitted') - self.tilted_efficiency = True - self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', 'Efficiency tilt'] - self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, self.tilt] - self.parameter_errors = [max([0.1, 0.1*p]) for p in self.initial_values] - self.fixes = [self.fix_endpoint, self.fix_background, self.fix_nu_mass, self.fix_amplitude, self.fix_scatter_ratio_b, self.fix_scatter_ratio_c, self.fix_tilt] - self.limits = [energy_limits, - [1e-10, None], - neutrino_limits, - [0, None], - [0.1, 1.], - [0.1, 1.], - [-0.5, 0.5]] - - - if self.plot_lineshape: - logger.info('Parameters: {}'.format(self.parameter_names)) - logger.info('Fixed: {}'.format(self.fixes)) - logger.info('Initial values: {}'.format(self.initial_values)) - - #if len(self.constrained_parameters) > 0: - #logger.info('{}\n{}'.format(self.constrained_parameters, self.parameter_names)) - #self.constrained_parameters = [self.parameter_names.index(p) for p in self.constrained_parameter_names[0]] - #self.print_level=0 - + #if not self.fit_efficiency_tilt: + self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', + 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', + 'res', 'two_gaussia_sigma_1', 'two_gaussian_sigma_2'] + self.initial_values = [self.endpoint, self.background, self.mass_guess**2, self.counts_guess, + self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, + self.res, self.two_gaussian_sigma_1, self.two_gaussian_sigma_2] + + self.fixes = [self.fix_endpoint, self.fix_background, self.fix_nu_mass, self.fix_amplitude, + self.fix_scatter_peak_ratio_b, self.fix_scatter_peak_ratio_c, + self.fix_res, self.fix_two_gaussian_sigma_1, self.fix_two_gaussian_sigma_2] + + self.limits = [energy_limits, + [0, None], + neutrino_limits, + [100, None], + [0.1, 1.], + [0.1, 1.], + [30, 100], + [1, 100], + [1, 100]] + + + + self.parameter_errors = [max([0.1, 0.1*p]) for p in self.initial_values] + + + # else: + # logger.warning('Efficiency tilt will be fitted') + # self.tilted_efficiency = True + # self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', 'Efficiency tilt'] + # self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, self.tilt] + # self.parameter_errors = [max([0.1, 0.1*p]) for p in self.initial_values] + # self.fixes = [self.fix_endpoint, self.fix_background, self.fix_nu_mass, self.fix_amplitude, self.fix_scatter_ratio_b, self.fix_scatter_ratio_c, self.fix_tilt] + # self.limits = [energy_limits, + # [1e-10, None], + # neutrino_limits, + # [0, None], + # [0.1, 1.], + # [0.1, 1.], + # [-0.5, 0.5]] + + + # if self.plot_lineshape: + # logger.info('Parameters: {}'.format(self.parameter_names)) + # logger.info('Fixed: {}'.format(self.fixes)) + # logger.info('Initial values: {}'.format(self.initial_values)) return True - # parameter sampling + # ========================================================================= + # Get random sample from normal, beta, or gamma distribution + # ========================================================================= def Gaussian_sample(self, mean, width): np.random.seed() return np.random.randn()*width+mean @@ -484,7 +610,88 @@ def Gamma_sample(self, mean, width): return np.random.gamma(a, 1/b) - ############################ conversion methods ########################### + def SamplePriors(self, sampled_parameters): + + for k in sampled_parameters.keys(): + if k in self.nuisance_parameters and self.nuisance_parameters[k]: + raise ValueError('{} is nuisance parameter.'.format(k)) + + logger.info('Sampling: {}'.format([k for k in sampled_parameters.keys() if sampled_parameters[k]])) + self.parameter_samples = {} + sample_values = [] + if 'res' in sampled_parameters.keys() and sampled_parameters['res']: + self.res = self.Gaussian_sample(self.res_mean, self.res_width) + if self.res <= 30.01/float(2*np.sqrt(2*np.log(2))): + logger.warning('Sampled resolution small. Setting to {}'.format(30.01/float(2*np.sqrt(2*np.log(2))))) + self.res = 30.01/float(2*np.sqrt(2*np.log(2))) + self.parameter_samples['res'] = self.res + sample_values.append(self.res) + if 'two_gaussian_sigma_1' in sampled_parameters.keys() and sampled_parameters['two_gaussian_sigma_1']: + self.two_gaussian_sigma_1 = self.Gaussian_sample(self.two_gaussian_sigma_1_mean, self.two_gaussian_sigma_1_width) + self.parameter_samples['two_gaussian_sigma_1'] = self.two_gaussian_sigma_1 + sample_values.append(self.two_gaussian_sigma_1) + if 'two_gaussian_sigma_2' in sampled_parameters.keys() and sampled_parameters['two_gaussian_sigma_2']: + self.two_gaussian_sigma_2 = self.Gaussian_sample(self.two_gaussian_sigma_2_mean, self.two_gaussian_sigma_2_width) + self.parameter_samples['two_gaussian_sigma_2'] = self.two_gaussian_sigma_2 + sample_values.append(self.two_gaussian_sigma_2) + if 'scatter_peak_ratio' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio']: + self.scatter_peak_ratio_b = self.Beta_sample(self.scatter_peak_ratio_mean, self.scatter_peak_ratio_width) + self.scatter_peak_ratio_c = 1 + self.fix_scatter_ratio_b = True + self.fix_scatter_ratio_c = True + self.parameter_samples['scatter_peak_ratio'] = self.scatter_peak_ratio_b + sample_values.append(self.scatter_peak_ratio_b) + + if self.correlated_b_c_scale and 'scatter_peak_ratio_b' in sampled_parameters.keys() and 'scatter_peak_ratio_c' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_b'] and sampled_parameters['scatter_peak_ratio_c']: + logger.info('Correlated b, c, scale sampling') + correlated_vars = np.random.multivariate_normal([self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_c_mean, self.scale_mean], self.b_c_scale_cov_matrix) + self.scatter_peak_ratio_b = correlated_vars[0] + self.scatter_peak_ratio_c = correlated_vars[1] + self.width_scaling = correlated_vars[2] + + self.fix_scatter_ratio_b = True + self.parameter_samples['scatter_peak_ratio_b'] = self.scatter_peak_ratio_b + sample_values.append(self.scatter_peak_ratio_b) + + self.parameter_samples['scatter_peak_ratio_c'] = self.scatter_peak_ratio_c + sample_values.append(self.scatter_peak_ratio_c) + self.fix_scatter_ratio_c = True + + else: + + if 'scatter_peak_ratio_b' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_b']: + logger.info('Uncorrelated b, c, scale sampling') + self.scatter_peak_ratio_b = self.Gamma_sample(self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_b_width) + self.fix_scatter_ratio_b = True + self.parameter_samples['scatter_peak_ratio_b'] = self.scatter_peak_ratio_b + sample_values.append(self.scatter_peak_ratio_b) + if 'scatter_peak_ratio_c' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_c']: + self.scatter_peak_ratio_c = self.Gamma_sample(self.scatter_peak_ratio_c_mean, self.scatter_peak_ratio_c_width) + self.parameter_samples['scatter_peak_ratio_c'] = self.scatter_peak_ratio_c + sample_values.append(self.scatter_peak_ratio_c) + self.fix_scatter_ratio_c = True + + if 'B' in sampled_parameters.keys() and sampled_parameters['B']: + self.B = self.Gaussian_sample(self.B_mean, self.B_width) + self.parameter_samples['B'] = self.B + sample_values.append(self.B) + if 'endpoint' in sampled_parameters.keys() and sampled_parameters['endpoint']: + self.endpoint = self.Gaussian_sample(self.endpoint_mean, self.endpoint_width) + self.parameter_samples['endpoint'] = self.endpoint + self.fix_endpoint = True + sample_values.append(self.endpoint) + + logger.info('Samples are: {}'.format(sample_values)) + #logger.info('Fit parameters: \n{}\nFixed: {}'.format(self.parameter_names, self.fixes)) + # set new values in model + self.ConfigureFit() + + return self.parameter_samples + + + # ========================================================================= + # Frequency - Energy conversion + # ========================================================================= def Energy(self, f, mixfreq=0.): """ @@ -510,8 +717,9 @@ def Frequency(self, E, Theta=None): - ################################# bins and data ######################################## - + # ========================================================================= + # Bin and energy array set&get + # ========================================================================= @property def energies(self): @@ -579,6 +787,10 @@ def ReSetBins(self): + # ========================================================================= + # Data generation and histogramming + # ========================================================================= + def GenerateData(self, params, N): #print('Generating data') @@ -637,86 +849,16 @@ def ConvertAndHistogram(self, weights=None): return self.hist - def SamplePriors(self, sampled_parameters): - - for k in sampled_parameters.keys(): - if k in self.nuisance_parameters and self.nuisance_parameters[k]: - raise ValueError('{} is nuisance parameter.'.format(k)) - - logger.info('Sampling: {}'.format([k for k in sampled_parameters.keys() if sampled_parameters[k]])) - self.parameter_samples = {} - sample_values = [] - if 'res' in sampled_parameters.keys() and sampled_parameters['res']: - self.res = self.Gaussian_sample(self.res_mean, self.res_width) - if self.res <= 30.01/float(2*np.sqrt(2*np.log(2))): - logger.warning('Sampled resolution small. Setting to {}'.format(30.01/float(2*np.sqrt(2*np.log(2))))) - self.res = 30.01/float(2*np.sqrt(2*np.log(2))) - self.parameter_samples['res'] = self.res - sample_values.append(self.res) - if 'two_gaussian_sigma_1' in sampled_parameters.keys() and sampled_parameters['two_gaussian_sigma_1']: - self.two_gaussian_sigma_1 = self.Gaussian_sample(self.two_gaussian_sigma_1_mean, self.two_gaussian_sigma_1_width) - self.parameter_samples['two_gaussian_sigma_1'] = self.two_gaussian_sigma_1 - sample_values.append(self.two_gaussian_sigma_1) - if 'two_gaussian_sigma_2' in sampled_parameters.keys() and sampled_parameters['two_gaussian_sigma_2']: - self.two_gaussian_sigma_2 = self.Gaussian_sample(self.two_gaussian_sigma_2_mean, self.two_gaussian_sigma_2_width) - self.parameter_samples['two_gaussian_sigma_2'] = self.two_gaussian_sigma_2 - sample_values.append(self.two_gaussian_sigma_2) - if 'scatter_peak_ratio' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio']: - self.scatter_peak_ratio_b = self.Beta_sample(self.scatter_peak_ratio_mean, self.scatter_peak_ratio_width) - self.scatter_peak_ratio_c = 1 - self.fix_scatter_ratio_b = True - self.fix_scatter_ratio_c = True - self.parameter_samples['scatter_peak_ratio'] = self.scatter_peak_ratio_b - sample_values.append(self.scatter_peak_ratio_b) - - if self.correlated_b_c_scale and 'scatter_peak_ratio_b' in sampled_parameters.keys() and 'scatter_peak_ratio_c' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_b'] and sampled_parameters['scatter_peak_ratio_c']: - logger.info('Correlated b, c, scale sampling') - correlated_vars = np.random.multivariate_normal([self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_c_mean, self.scale_mean], self.b_c_scale_cov_matrix) - self.scatter_peak_ratio_b = correlated_vars[0] - self.scatter_peak_ratio_c = correlated_vars[1] - self.width_scaling = correlated_vars[2] - - self.fix_scatter_ratio_b = True - self.parameter_samples['scatter_peak_ratio_b'] = self.scatter_peak_ratio_b - sample_values.append(self.scatter_peak_ratio_b) - - self.parameter_samples['scatter_peak_ratio_c'] = self.scatter_peak_ratio_c - sample_values.append(self.scatter_peak_ratio_c) - self.fix_scatter_ratio_c = True - - else: - - if 'scatter_peak_ratio_b' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_b']: - logger.info('Uncorrelated b, c, scale sampling') - self.scatter_peak_ratio_b = self.Gamma_sample(self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_b_width) - self.fix_scatter_ratio_b = True - self.parameter_samples['scatter_peak_ratio_b'] = self.scatter_peak_ratio_b - sample_values.append(self.scatter_peak_ratio_b) - if 'scatter_peak_ratio_c' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_c']: - self.scatter_peak_ratio_c = self.Gamma_sample(self.scatter_peak_ratio_c_mean, self.scatter_peak_ratio_c_width) - self.parameter_samples['scatter_peak_ratio_c'] = self.scatter_peak_ratio_c - sample_values.append(self.scatter_peak_ratio_c) - self.fix_scatter_ratio_c = True - - if 'B' in sampled_parameters.keys() and sampled_parameters['B']: - self.B = self.Gaussian_sample(self.B_mean, self.B_width) - self.parameter_samples['B'] = self.B - sample_values.append(self.B) - if 'endpoint' in sampled_parameters.keys() and sampled_parameters['endpoint']: - self.endpoint = self.Gaussian_sample(self.endpoint_mean, self.endpoint_width) - self.parameter_samples['endpoint'] = self.endpoint - self.fix_endpoint = True - sample_values.append(self.endpoint) - - logger.info('Samples are: {}'.format(sample_values)) - #logger.info('Fit parameters: \n{}\nFixed: {}'.format(self.parameter_names, self.fixes)) - # set new values in model - self.ConfigureFit() - return self.parameter_samples + # ========================================================================= + # Fit + # ========================================================================= + # This is the main function that is called from outside (besides Configure). + # Maybe I should rename it to InternalRun def SampleConvertAndFit(self, sampled_parameters={}, params= []): + # for systematic MC uncertainty propagation: Generate Asimov data to fit with random model if self.use_asimov: #temp = self.error_scaling #self.error_scaling = 0 @@ -724,10 +866,10 @@ def SampleConvertAndFit(self, sampled_parameters={}, params= []): self.hist = self.TritiumSpectrumBackground(self.bin_centers, *params) #self.error_scaling = temp - # if random_priors contains 3 items (all boolean) get new sample froom priors + # sample priors that are to be sampled if len(sampled_parameters.keys()) > 0: self.SamplePriors(sampled_parameters) - # re-calculate bin efficiencies, if self.pseudo_eff=True efficiency will be ranomized + # need to first re-calcualte energy bins with sampled B before getting efficiency self.ReSetBins() @@ -736,6 +878,8 @@ def SampleConvertAndFit(self, sampled_parameters={}, params= []): random_efficiency = True else: random_efficiency = False + + # re-calculate bin efficiencies, if self.pseudo_eff=True efficiency will be ranomized self._bin_efficiency, self._bin_efficiency_error = self.Efficiency(self.bin_centers, pseudo=random_efficiency) @@ -743,8 +887,7 @@ def SampleConvertAndFit(self, sampled_parameters={}, params= []): if not self.use_asimov: self.ConvertAndHistogram() - - + # Call parent fit method using data and model from this instance return self.fit() @@ -1011,6 +1154,13 @@ def running_mean(self, x, N): def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=None, res=None, sig1=None, sig2=None, tilt=0., error=False): E = np.array(E) + if sig1 == None: + sig1 = self.two_gaussian_sigma_1 + if sig2 == None: + sig2 = self.two_gaussian_sigma_2 + if res == None: + res = self.res + if len(E)==0: E = self._bin_centers @@ -1047,7 +1197,7 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No if self.resolution_model != 'two_gaussian': lineshape = self.gauss_resolution_f(e_lineshape, 1, res*self.width_scaling, 0) else: - lineshape = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, two_gaussian_sigma_1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, two_gaussian_sigma_2*self.width_scaling, self.two_gaussian_mu_2) + lineshape = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, sig1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, sig2*self.width_scaling, self.two_gaussian_mu_2) # spectrum shape spec = self.which_model(e_spec, endpoint, m_nu) @@ -1069,7 +1219,6 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No # simplified lineshape - FWHM = 2.*np.sqrt(2.*np.log(2.))*res *self.width_scaling if FWHM < 30.01: #logger.warning('FWHM smaller 30. Setting to 30 instead.') @@ -1362,131 +1511,3 @@ def Efficiency(self, E, freq=False, pseudo=False): -######################################################### -# Functions using model above - - -def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, - i=0, fixed_data = [], error_scaling=1, tilt = None, fit_tilt = False, event=None): - - """if 'scattered' in fit_options.keys(): - scattered = fit_options['scattered'] - else: - scattered = None - if 'distorted' in fit_options.keys(): - distorted = fit_options['distorted'] - else: - distorted = None - if 'fit_nu_mass' in fit_options.keys(): - fit_nu_mass = fit_options['fit_nu_mass'] - else: - fit_nu_mass = None""" - - if i%20 == 0: - logger.info('Sampling: {}'.format(i)) - - - - try: - - T = BinnedTritiumMLFitter("TritiumFitter") - T.InternalConfigure(fit_config_dict) - #T.is_scattered = scattered - #T.is_distorted = distorted - T.error_scaling = error_scaling - T.integrate_bins = True - - if tilt is not None:# and tilt !=0: - T.tilted_efficiency = True - T.tilt = tilt - - - - # generate random data from best fit parameters - if len(fixed_data) == 0: - _, new_data = T.GenerateData(params, counts) - - else: - _, new_data = T.GenerateAsimovData(params) - - - #if fit_nu_mass: - # T.fix_nu_mass = False - - if fit_tilt: - T.fix_tilt = False - - T.freq_data = new_data - results, errors = T.SampleConvertAndFit(sampled_priors, params) - parameter_samples = T.parameter_samples - - except Exception as e: - print(e) - print(params) - if event is not None: - event.set() - raise(e) - - else: - if fit_config_dict['minos_intervals']: - return results, T.minos_errors, parameter_samples - else: - return results, errors, parameter_samples - - -def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, - tilt=None, fit_tilt = False, data_is_energy=False): - - """scattered = fit_options['scattered'] - distorted = fit_options['distorted'] - fit_nu_mass = fit_options['fit_nu_mass']""" - - T = BinnedTritiumMLFitter("TritiumFitter") - T.InternalConfigure(fit_config_dict) - T.print_level=1 - #T.is_scattered = scattered - #T.is_distorted = distorted - T.error_scaling = error_scaling - T.integrate_bins = True - logger.info('Energy stepsize: {}'.format(T.denergy)) - - if tilt is not None:# and tilt != 0: - T.tilted_efficiency = True - T.tilt = tilt - - #if fit_nu_mass: - # logger.info('Fitting neutrino mass') - # T.fix_nu_mass = False - if fit_tilt: - logger.info('Going to fit efficiency tilt') - T.fix_tilt = False - - if data_is_energy: - data = T.Frequency(data) - T.freq_data = data - results, errors = T.SampleConvertAndFit(sampled_parameters) - total_counts = results[1]+results[3] - - if fit_config_dict['minos_intervals']: - return results, T.minos_errors, total_counts - else: - return results, errors, total_counts - -def GetPDF(fit_config_dict, params, plot=False): - logger.info('Plotting lineshape: {}'.format(plot)) - logger.info('PDF for params: {}'.format(params)) - #logger.info(fit_options) - #scattered = fit_options['scattered'] - #distorted = fit_options['distorted'] - - T = BinnedTritiumMLFitter("TritiumFitter") - T.InternalConfigure(fit_config_dict) - T.plot_lineshape = plot - #T.is_scatterd=scattered - #T.is_distorted=distorted - - pdf = T.TritiumSpectrumBackground(T.energies, *params) - _, asimov_binned_data = T.GenerateAsimovData(params) - binned_fit = T.TritiumSpectrumBackground(T.bin_centers, *params, error=True) - - return T.energies, pdf, T.bin_centers, binned_fit, asimov_binned_data diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 364803ec..d94d4ea7 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -425,6 +425,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 ratesS[ratesS<0.] = 0. ratesB[ratesB<0.] = 0. + logger.info('Combinging {} signal events with {} background events'.format(S, B)) rate_sumS, rate_sumB = np.sum(ratesS), np.sum(ratesB) probsS = np.array(ratesS)/rate_sumS probsB = np.array(ratesB)/rate_sumB From 59ca535c65c890133bdbfc4e3f919067257ad7fc Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 13 Oct 2021 16:55:20 -0700 Subject: [PATCH 126/199] more reorganization --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 131 +++++++----------- 1 file changed, 52 insertions(+), 79 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 58e9620e..aabe49f1 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -3,14 +3,14 @@ Author: C. Claessens Date:8/03/2021 Description: - Tritium class + Processor for analyzing tritium data with frequentist methods contains tritium model(s) convolves with energy resolution convolves with lineshape multiplies with efficiency adds background - has log likelihood and neg log likelihood functions (poisson) that can be used by minimzer + uses BinnedDataFitter to fit binned spectrum with iminuit minimizer """ @@ -38,8 +38,10 @@ #electron_mass = constants.electron_mass/constants.e*constants.c**2 #FineStructureConstant = 0.0072973525664 -############################################################################### +# ============================================================================= # Functions that can be imported from here and facitilate working with the processor +# ============================================================================= +# # They create an instance of the processor defined below and use it for analysis # GenAndFit tells the processor to generate new fake data and fit it # DoOneFit give the processor data and tells it to fit it @@ -50,30 +52,14 @@ def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, i=0, fixed_data = [], error_scaling=1, tilt = None, fit_tilt = False, event=None): - """if 'scattered' in fit_options.keys(): - scattered = fit_options['scattered'] - else: - scattered = None - if 'distorted' in fit_options.keys(): - distorted = fit_options['distorted'] - else: - distorted = None - if 'fit_nu_mass' in fit_options.keys(): - fit_nu_mass = fit_options['fit_nu_mass'] - else: - fit_nu_mass = None""" if i%20 == 0: logger.info('Sampling: {}'.format(i)) - - try: T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) - #T.is_scattered = scattered - #T.is_distorted = distorted T.error_scaling = error_scaling T.integrate_bins = True @@ -82,7 +68,6 @@ def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, T.tilt = tilt - # generate random data from best fit parameters if len(fixed_data) == 0: _, new_data = T.GenerateData(params, counts) @@ -91,9 +76,6 @@ def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, _, new_data = T.GenerateAsimovData(params) - #if fit_nu_mass: - # T.fix_nu_mass = False - if fit_tilt: T.fix_tilt = False @@ -104,8 +86,6 @@ def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, except Exception as e: print(e) print(params) - if event is not None: - event.set() raise(e) else: @@ -118,15 +98,10 @@ def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, tilt=None, fit_tilt = False, data_is_energy=False): - """scattered = fit_options['scattered'] - distorted = fit_options['distorted'] - fit_nu_mass = fit_options['fit_nu_mass']""" T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) T.print_level=1 - #T.is_scattered = scattered - #T.is_distorted = distorted T.error_scaling = error_scaling T.integrate_bins = True logger.info('Energy stepsize: {}'.format(T.denergy)) @@ -135,9 +110,6 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, T.tilted_efficiency = True T.tilt = tilt - #if fit_nu_mass: - # logger.info('Fitting neutrino mass') - # T.fix_nu_mass = False if fit_tilt: logger.info('Going to fit efficiency tilt') T.fix_tilt = False @@ -156,15 +128,12 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, def GetPDF(fit_config_dict, params, plot=False): logger.info('Plotting lineshape: {}'.format(plot)) logger.info('PDF for params: {}'.format(params)) - #logger.info(fit_options) - #scattered = fit_options['scattered'] - #distorted = fit_options['distorted'] + T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) T.plot_lineshape = plot - #T.is_scatterd=scattered - #T.is_distorted=distorted + pdf = T.TritiumSpectrumBackground(T.energies, *params) _, asimov_binned_data = T.GenerateAsimovData(params) @@ -173,8 +142,9 @@ def GetPDF(fit_config_dict, params, plot=False): return T.energies, pdf, T.bin_centers, binned_fit, asimov_binned_data -############################################################################## +# ============================================================================= # Processor definition +# ============================================================================= class BinnedTritiumMLFitter(BinnedDataFitter): @@ -537,7 +507,6 @@ def ConfigureFit(self): logger.warning('Neutrino mass limited to: {} - {}'.format(*np.sqrt(np.array(neutrino_limits)))) - #logger.warning('Neutrino mass fitted: {}'.format(self.fit_nu_mass)) #if not self.fit_efficiency_tilt: self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', @@ -589,6 +558,46 @@ def ConfigureFit(self): return True + # ========================================================================= + # Main fit function + # ========================================================================= + # This is the main function that is called from outside (besides Configure). + # Maybe I should rename it to InternalRun to match mermithid naming scheme (and make the processor an actual processor) + + def SampleConvertAndFit(self, sampled_parameters={}, params= []): + + # for systematic MC uncertainty propagation: Generate Asimov data to fit with random model + if self.use_asimov: + #temp = self.error_scaling + #self.error_scaling = 0 + #self._bin_efficiency, self._bin_efficiency_error = self.Efficiency(self.bin_centers, pseudo=True) + self.hist = self.TritiumSpectrumBackground(self.bin_centers, *params) + #self.error_scaling = temp + + # sample priors that are to be sampled + if len(sampled_parameters.keys()) > 0: + self.SamplePriors(sampled_parameters) + + + # need to first re-calcualte energy bins with sampled B before getting efficiency + self.ReSetBins() + + if 'efficiency' in sampled_parameters.keys(): + random_efficiency = True + else: + random_efficiency = False + + # re-calculate bin efficiencies, if self.pseudo_eff=True efficiency will be ranomized + self._bin_efficiency, self._bin_efficiency_error = self.Efficiency(self.bin_centers, pseudo=random_efficiency) + + + # now convert frequency data to energy data and histogram it + if not self.use_asimov: + self.ConvertAndHistogram() + + # Call parent fit method using data and model from this instance + return self.fit() + # ========================================================================= # Get random sample from normal, beta, or gamma distribution @@ -850,50 +859,14 @@ def ConvertAndHistogram(self, weights=None): - # ========================================================================= - # Fit - # ========================================================================= - # This is the main function that is called from outside (besides Configure). - # Maybe I should rename it to InternalRun - - def SampleConvertAndFit(self, sampled_parameters={}, params= []): - - # for systematic MC uncertainty propagation: Generate Asimov data to fit with random model - if self.use_asimov: - #temp = self.error_scaling - #self.error_scaling = 0 - #self._bin_efficiency, self._bin_efficiency_error = self.Efficiency(self.bin_centers, pseudo=True) - self.hist = self.TritiumSpectrumBackground(self.bin_centers, *params) - #self.error_scaling = temp - - # sample priors that are to be sampled - if len(sampled_parameters.keys()) > 0: - self.SamplePriors(sampled_parameters) - - - # need to first re-calcualte energy bins with sampled B before getting efficiency - self.ReSetBins() - - if 'efficiency' in sampled_parameters.keys(): - random_efficiency = True - else: - random_efficiency = False - # re-calculate bin efficiencies, if self.pseudo_eff=True efficiency will be ranomized - self._bin_efficiency, self._bin_efficiency_error = self.Efficiency(self.bin_centers, pseudo=random_efficiency) - - - # now convert frequency data to energy data and histogram it - if not self.use_asimov: - self.ConvertAndHistogram() - - # Call parent fit method using data and model from this instance - return self.fit() - ########################### Tritium spectrum ################################# + # ========================================================================= + # Model functions + # ========================================================================= def gauss_resolution_f(self, energy_array, A, sigma, mu): From 9aed766b2f0086c7810216ac0ae7a86f767070f7 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 14 Oct 2021 00:00:31 -0700 Subject: [PATCH 127/199] made model parameters configurable --- .../misc/DetectionEfficiencyUtilities.py | 23 +- .../Fitters/MCUncertaintyPropagation.py | 20 +- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 503 ++++++++++-------- 3 files changed, 302 insertions(+), 244 deletions(-) diff --git a/mermithid/misc/DetectionEfficiencyUtilities.py b/mermithid/misc/DetectionEfficiencyUtilities.py index 6a6c7ab9..475d363d 100755 --- a/mermithid/misc/DetectionEfficiencyUtilities.py +++ b/mermithid/misc/DetectionEfficiencyUtilities.py @@ -6,12 +6,12 @@ from scipy.interpolate import interp1d from scipy.stats import binom from scipy import constants -from statsmodels.stats.proportion import proportion_confint import helper_functions.alis_power as ap import helper_functions.plot_methods as pm def binomial_interval(n, k, alpha=1 / 3.1514872): + from statsmodels.stats.proportion import proportion_confint if isinstance(n, list) or type(n) is np.ndarray: @@ -257,24 +257,3 @@ def pseudo_integrated_efficiency(f, df, snr_efficiency_dict, alpha=1): return pseudo_y+y, y_error -def binomial_interval(n, k, alpha=1 / 3.1514872): - - if isinstance(n, list) or type(n) is np.ndarray: - - mean = np.zeros(len(n)) - interval = np.zeros((2, len(n))) - interval_mean = np.zeros(len(n)) - for i in range(len(n)): - interval[0][i], interval[1][i] = proportion_confint(k[i], n[i], method='wilson', alpha=alpha) - rv = binom(n[i], k[i]*1./n[i]) - mean[i] = rv.mean()*1./n[i] - interval_mean[i] = np.mean(interval.T[i]) - interval_error = [interval_mean - interval[0], interval[1]-interval_mean] - - else: - l_int, u_int = proportion_confint(k, n, method='wilson', alpha=alpha) - rv = binom(n, k*1./n) - mean = rv.mean()*1./n - interval_mean = np.mean([l_int, u_int]) - interval_error = [interval_mean - l_int, u_int-interval_mean] - return mean, interval_mean, interval_error \ No newline at end of file diff --git a/mermithid/processors/Fitters/MCUncertaintyPropagation.py b/mermithid/processors/Fitters/MCUncertaintyPropagation.py index 78a0cd0d..3ab6aa94 100644 --- a/mermithid/processors/Fitters/MCUncertaintyPropagation.py +++ b/mermithid/processors/Fitters/MCUncertaintyPropagation.py @@ -74,17 +74,17 @@ def InitialFit(self): fit_successful = False counter = 0 - while (not fit_successful) and counter < 15: - counter += 1 - try: + #while (not fit_successful) and counter < 15: + # counter += 1 + # try: - self.fitted_params, self.fitted_params_errors, self.Counts = self.fit(self.data, + self.fitted_params, self.fitted_params_errors, self.Counts = self.fit(self.data, self.fit_config_dict) - fit_successful = True - except Exception as e: - print(e) - logger.error('Repeating fit') - continue + fit_successful = True + # except Exception as e: + # print(e) + # logger.error('Repeating fit') + # continue logger.info('Best fit: {}'.format(self.fitted_params)) @@ -124,7 +124,7 @@ def ParameterSampling(self): for i in range(start_j, self.N): fit_successful = False counter = 0 - while (not fit_successful) and counter < 15: + while (not fit_successful) and counter < 5: counter += 1 try: all_fit_returns.append(self.gen_and_fit(self.fitted_params, self.Counts, diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index aabe49f1..29d4bee1 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -56,43 +56,38 @@ def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, if i%20 == 0: logger.info('Sampling: {}'.format(i)) - try: - T = BinnedTritiumMLFitter("TritiumFitter") - T.InternalConfigure(fit_config_dict) - T.error_scaling = error_scaling - T.integrate_bins = True - if tilt is not None:# and tilt !=0: - T.tilted_efficiency = True - T.tilt = tilt + T = BinnedTritiumMLFitter("TritiumFitter") + T.InternalConfigure(fit_config_dict) + #T.error_scaling = error_scaling + T.integrate_bins = True + if tilt is not None:# and tilt !=0: + T.tilted_efficiency = True + T.tilt = tilt - # generate random data from best fit parameters - if len(fixed_data) == 0: - _, new_data = T.GenerateData(params, counts) - else: - _, new_data = T.GenerateAsimovData(params) + # generate random data from best fit parameters + if len(fixed_data) == 0: + _, new_data = T.GenerateData(params, counts) + else: + _, new_data = T.GenerateAsimovData(params) - if fit_tilt: - T.fix_tilt = False - T.freq_data = new_data - results, errors = T.SampleConvertAndFit(sampled_priors, params) - parameter_samples = T.parameter_samples + if fit_tilt: + T.fix_tilt = False + + T.freq_data = new_data + results, errors = T.SampleConvertAndFit(sampled_priors, params) + parameter_samples = T.parameter_samples - except Exception as e: - print(e) - print(params) - raise(e) + if fit_config_dict['minos_intervals']: + return results, T.minos_errors, parameter_samples else: - if fit_config_dict['minos_intervals']: - return results, T.minos_errors, parameter_samples - else: - return results, errors, parameter_samples + return results, errors, parameter_samples def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, @@ -102,7 +97,7 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) T.print_level=1 - T.error_scaling = error_scaling + #T.error_scaling = error_scaling T.integrate_bins = True logger.info('Energy stepsize: {}'.format(T.denergy)) @@ -133,11 +128,12 @@ def GetPDF(fit_config_dict, params, plot=False): T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) T.plot_lineshape = plot + T.pass_efficiency_error = True pdf = T.TritiumSpectrumBackground(T.energies, *params) _, asimov_binned_data = T.GenerateAsimovData(params) - binned_fit = T.TritiumSpectrumBackground(T.bin_centers, *params, error=True) + binned_fit = T.TritiumSpectrumBackground(T.bin_centers, *params) return T.energies, pdf, T.bin_centers, binned_fit, asimov_binned_data @@ -156,31 +152,148 @@ def InternalConfigure(self, config_dict): # ==================================== self.use_approx_model = reader.read_param(config_dict, 'use_approximate_model', True) self.use_toy_model_efficiency = reader.read_param(config_dict, 'use_toy_model_efficiency', False) - self.fit_nu_mass = reader.read_param(config_dict, 'fit_neutrino_mass', False) self.is_distorted = reader.read_param(config_dict, 'distorted', False) # multiply spectrum with efficency self.is_smeared = reader.read_param(config_dict, 'smeared', True) # convolve with energy resolution self.error_scaling = reader.read_param(config_dict, 'error_scaling', 1.) # scales efficiency uncertainty self.is_scattered = reader.read_param(config_dict, 'scattered', False) # convolve with lineshape self.integrate_bins = reader.read_param(config_dict, 'integrate_bins', True) # integrate spectrum over bin widths self.fit_efficiency_tilt = reader.read_param(config_dict, 'fit_efficiency_tilt', False) # efficiency slope is free parameter + self.fit_nu_mass = reader.read_param(config_dict, 'fit_neutrino_mass', False) + + # detector response options + self.NScatters = reader.read_param(config_dict, 'NScatters', 20) + self.resolution_model = reader.read_param(config_dict, 'resolution_model', 'gaussian') + self.lineshape_model = reader.read_param(config_dict, 'lineshape_model', 'simplified') + self.simplified_lineshape_path = reader.read_param(config_dict, 'simplified_lineshape_path', "required") + self.helium_lineshape_path = reader.read_param(config_dict, 'helium_lineshape_path', "optional") + self.hydrogen_proportion = reader.read_param(config_dict, 'hydrogen_proportion', 1) + self.use_helium_scattering = reader.read_param(config_dict, 'use_helium_scattering', False) + self.correlated_b_c_scale = reader.read_param(config_dict, 'correlated_b_c_scale', False) + + + # configure model parameter names + self.model_parameter_names = reader.read_param(config_dict, 'model_parameter_names', + ['endpoint', 'background', 'm_beta_squared', 'Amplitude', + 'scatter_peak_ratio_p', 'scatter_peak_ratio_q', + 'resolution', 'two_gaussian_sigma_1', 'two_gaussian_sigma_2'] ) + # initial values and mean of constaints (if constraint) or mean of distribution (if sampled) + self.model_parameter_means = reader.read_param(config_dict, 'model_parameter_means', [18.6e3, 0, 0, 5000, 0.8, 1, 15, 10, 10]) + # width for constraints or sample distributions + self.model_parameter_widths = reader.read_param(config_dict, 'model_parameter_widths', [100, 0.1, 0.1, 500, 0.1, 0, 3, 1, 1]) + self.fixed_parameters = reader.read_param(config_dict, 'fixed_parameters', [False, False, False, False, True, True, True, True, True]) + self.limits = reader.read_param(config_dict, 'model_parameter_limits', + [[18e3, 20e3], + [0, None], + [-300**2, 300**2], + [100, None], + [0.1, 1.], + [0.1, 1.], + [30, 100], + [1, 100], + [1, 100]]) + + + # check that configuration is consistent + if (len(self.model_parameter_names) != len(self.model_parameter_means) or len(self.model_parameter_names) != len(self.model_parameter_widths) or len(self.model_parameter_names) != len(self.fixed_parameters)): + raise IndexError('Number of parameter names does not match other parameter configurations') + + + # ==================================== + # Identify and configure parameters + # ==================================== + + self.parameter_samples = {} + + + self.background_index = self.model_parameter_names.index('background') + # and which one is B + # self.B_index = self.model_parameter_names.index('B') + # signal counts + self.amplitude_index = self.model_parameter_names.index('Amplitude') + + + # scatter peak ratio + if self.is_scattered: + self.scatter_peak_ratio_b_index = self.model_parameter_names.index('scatter_peak_ratio_p') + self.scatter_peak_ratio_c_index = self.model_parameter_names.index('scatter_peak_ratio_q') + + self.scatter_peak_ratio_b_mean = reader.read_param(config_dict, 'scatter_peak_ratio_b_mean', 0.7) + self.scatter_peak_ratio_b_width = reader.read_param(config_dict, 'scatter_peak_ratio_b_width', 0.1) + self.scatter_peak_ratio_c_mean = reader.read_param(config_dict, 'scatter_peak_ratio_c_mean', 0.7) + self.scatter_peak_ratio_c_width = reader.read_param(config_dict, 'scatter_peak_ratio_c_width', 0.1) + + self.scatter_peak_ratio_mean = reader.read_param(config_dict, 'scatter_peak_ratio_mean', 0.5) + self.scatter_peak_ratio_width = reader.read_param(config_dict, 'scatter_peak_ratio_width', 0.1) + + self.scatter_peak_ratio_b = self.scatter_peak_ratio_b_mean + self.scatter_peak_ratio_c = self.scatter_peak_ratio_c_mean + + #Adding correlated parameters (will later incorporate into morpho processor) + self.b_c_corr = reader.read_param(config_dict, 'b_c_corr', 0) + self.b_scale_corr = reader.read_param(config_dict, 'b_scale_corr', 0) + self.c_scale_corr = reader.read_param(config_dict, 'c_scale_corr', 0) + + + stds = [self.scatter_peak_ratio_b_width, self.scatter_peak_ratio_c_width, self.scale_width] + self.b_c_scale_cov_matrix = [[stds[0]**2, self.b_c_corr*stds[0]*stds[1], self.b_scale_corr*stds[0]*stds[2]], + [self.b_c_corr*stds[1]*stds[0], stds[1]**2, self.c_scale_corr*stds[1]*stds[2]], + [self.b_scale_corr*stds[2]*stds[0], self.c_scale_corr*stds[2]*stds[1], stds[2]**2]] + + + # resolutions + if self.is_smeared: + self.res_index = self.model_parameter_names.index('resolution') + + self.res_mean = reader.read_param(config_dict, 'gaussian_resolution_mean', 15.0) + self.res_width = reader.read_param(config_dict, 'gaussian_resolution_width', 1.0) + self.res = self.res_mean + if self.res <= 30.01/float(2*np.sqrt(2*np.log(2))): + logger.warning('Resolution small for shallow trap model. Setting to {}'.format(30.01/float(2*np.sqrt(2*np.log(2))))) + self.res = 30.01/float(2*np.sqrt(2*np.log(2))) + + self.scale_mean = reader.read_param(config_dict, 'scale_mean', 1) + self.scale_width = reader.read_param(config_dict, 'scale_width', 0) + self.width_scaling = self.scale_mean + + if self.resolution_model != 'gaussian': + self.two_gaussian_sigma_1_index = self.model_parameter_names.index('two_gaussian_sigma_1') + self.two_gaussian_sigma_2_index = self.model_parameter_names.index('two_gaussian_sigma_2') + + self.two_gaussian_mu_1 = reader.read_param(config_dict, 'two_gaussian_mu1', 0) + self.two_gaussian_mu_2 = reader.read_param(config_dict, 'two_gaussian_mu2', 0) + self.two_gaussian_sigma_1_mean = reader.read_param(config_dict, 'two_gaussian_sigma_1_mean', 15) + self.two_gaussian_sigma_2_mean = reader.read_param(config_dict, 'two_gaussian_sigma_2_mean', 5) + self.two_gaussian_sigma_1_width = reader.read_param(config_dict, 'two_gaussian_sigma_1_width', 1) + self.two_gaussian_sigma_2_width = reader.read_param(config_dict, 'two_gaussian_sigma_2_width', 1) + self.two_gaussian_wide_fraction = reader.read_param(config_dict, 'two_gaussian_wide_fraction', 1.) + self.two_gaussian_sigma_1 = self.two_gaussian_sigma_1_mean + self.two_gaussian_sigma_2 = self.two_gaussian_sigma_2_mean + + # identify tritium parameters + self.tritium_model_indices = reader.read_param(config_dict, 'tritium_model_parameters', [0, 2]) + self.m_beta_index = self.model_parameter_names.index('m_beta_squared') + self.endpoint_index = self.model_parameter_names.index('endpoint') + self.fixed_parameters[self.m_beta_index] = not self.fit_nu_mass + self.endpoint=reader.read_param(config_dict, 'true_endpoint', 18.573e3) # final state spectrum self.use_final_states = reader.read_param(config_dict, 'use_final_states', False) self.final_state_array = reader.read_param(config_dict, 'final_states_array', [[0], [1]]) - self.use_asimov = False - # save plots in (processor can plot lineshape used in tritium model) self.savepath = reader.read_param(config_dict, 'savepath', '.') - # model parameters and uncertainties + # individual model parameter configurations + # overwrites model_parameter_means and widths self.B_mean = reader.read_param(config_dict, 'B_mean', 1) self.B_width = reader.read_param(config_dict, 'B_width', 1e-6) - self.endpoint_mean = reader.read_param(config_dict,'endpoint_mean', 18.6e3) - self.endpoint_width = reader.read_param(config_dict, 'endpoint_width', 1) + self.B = self.B_mean + + self.endpoint_mean = reader.read_param(config_dict,'endpoint_mean', self.model_parameter_means[self.endpoint_index]) + self.endpoint_width = reader.read_param(config_dict, 'endpoint_width', self.model_parameter_widths[self.endpoint_index]) # frequency range self.min_frequency = reader.read_param(config_dict, 'min_frequency', "required") @@ -190,50 +303,11 @@ def InternalConfigure(self, config_dict): # path to json with efficiency dictionary self.efficiency_file_path = reader.read_param(config_dict, 'efficiency_file_path', '') - # detector response - self.NScatters = reader.read_param(config_dict, 'NScatters', 20) - self.resolution_model = reader.read_param(config_dict, 'resolution_model', 'gaussian') - self.lineshape_model = reader.read_param(config_dict, 'lineshape_model', 'simplified') - self.simplified_lineshape_path = reader.read_param(config_dict, 'simplified_lineshape_path', "required") - self.helium_lineshape_path = reader.read_param(config_dict, 'helium_lineshape_path', "optional") - self.hydrogen_proportion = reader.read_param(config_dict, 'hydrogen_proportion', 1) - self.use_helium_scattering = reader.read_param(config_dict, 'use_helium_scattering', False) - - self.scatter_peak_ratio_b_mean = reader.read_param(config_dict, 'scatter_peak_ratio_b_mean', 0.7) - self.scatter_peak_ratio_b_width = reader.read_param(config_dict, 'scatter_peak_ratio_b_width', 0.1) - self.scatter_peak_ratio_c_mean = reader.read_param(config_dict, 'scatter_peak_ratio_c_mean', 0.7) - self.scatter_peak_ratio_c_width = reader.read_param(config_dict, 'scatter_peak_ratio_c_width', 0.1) - - self.scatter_peak_ratio_mean = reader.read_param(config_dict, 'scatter_peak_ratio_mean', 0.5) - self.scatter_peak_ratio_width = reader.read_param(config_dict, 'scatter_peak_ratio_width', 0.1) - - self.scale_mean = reader.read_param(config_dict, 'scale_mean', 1) - self.scale_width = reader.read_param(config_dict, 'scale_width', 0) - - - #Adding correlated parameters (will later incorporate into morpho processor) - self.b_c_corr = reader.read_param(config_dict, 'b_c_corr', 0) - self.b_scale_corr = reader.read_param(config_dict, 'b_scale_corr', 0) - self.c_scale_corr = reader.read_param(config_dict, 'c_scale_corr', 0) - self.correlated_b_c_scale = reader.read_param(config_dict, 'correlated_b_c_scale', False) - stds = [self.scatter_peak_ratio_b_width, self.scatter_peak_ratio_c_width, self.scale_width] - self.b_c_scale_cov_matrix = [[stds[0]**2, self.b_c_corr*stds[0]*stds[1], self.b_scale_corr*stds[0]*stds[2]], - [self.b_c_corr*stds[1]*stds[0], stds[1]**2, self.c_scale_corr*stds[1]*stds[2]], - [self.b_scale_corr*stds[2]*stds[0], self.c_scale_corr*stds[2]*stds[1], stds[2]**2]] - self.res_mean = reader.read_param(config_dict, 'gaussian_resolution_mean', 15.0) - self.res_width = reader.read_param(config_dict, 'gaussian_resolution_width', 1.0) - self.two_gaussian_mu_1 = reader.read_param(config_dict, 'two_gaussian_mu1', 0) - self.two_gaussian_mu_2 = reader.read_param(config_dict, 'two_gaussian_mu2', 0) - self.two_gaussian_sigma_1_mean = reader.read_param(config_dict, 'two_gaussian_sigma_1_mean', 15) - self.two_gaussian_sigma_2_mean = reader.read_param(config_dict, 'two_gaussian_sigma_2_mean', 5) - self.two_gaussian_sigma_1_width = reader.read_param(config_dict, 'two_gaussian_sigma_1_width', 1) - self.two_gaussian_sigma_2_width = reader.read_param(config_dict, 'two_gaussian_sigma_2_width', 1) - self.two_gaussian_wide_fraction = reader.read_param(config_dict, 'two_gaussian_wide_fraction', 1.) # ================= @@ -241,67 +315,82 @@ def InternalConfigure(self, config_dict): # ================= self.print_level = reader.read_param(config_dict, 'print_level', 0) - self.fix_nu_mass = not self.fit_nu_mass - self.fix_endpoint = not reader.read_param(config_dict, 'fit_endpoint', True) - self.fix_background = not reader.read_param(config_dict, 'fit_background', True) - self.fix_amplitude = not reader.read_param(config_dict, 'fit_amplitude', True) - self.counts_guess = reader.read_param(config_dict, 'counts_guess', 5000) - self.mass_guess = reader.read_param(config_dict, 'nu_mass_guess', 0.0) + #self.fix_nu_mass = not self.fit_nu_mass + self.pass_efficiency_error = False + self.use_asimov = False + #self.fix_endpoint = not reader.read_param(config_dict, 'fit_endpoint', True) + #self.fix_background = not reader.read_param(config_dict, 'fit_background', True) + #self.fix_amplitude = not reader.read_param(config_dict, 'fit_amplitude', True) + #self.counts_guess = reader.read_param(config_dict, 'counts_guess', 5000) + #self.mass_guess = reader.read_param(config_dict, 'nu_mass_guess', 0.0) + + # Parameters can be constrained manually or by inlcuding them in the nuisance parameter dictionary - self.constrained_parameter_names = reader.read_param(config_dict, 'constrained_parameter_names', []) + #self.constrained_parameter_names = reader.read_param(config_dict, 'constrained_parameter_names', []) self.constrained_parameters = reader.read_param(config_dict, 'constrained_parameters', []) - self.constrained_means = reader.read_param(config_dict, 'constrained_means', []) - self.constrained_widths = reader.read_param(config_dict, 'constrained_widths', []) + self.constrained_means = np.array(self.model_parameter_means)[self.constrained_parameters]#reader.read_param(config_dict, 'constrained_means', []) + self.constrained_widths = np.array(self.model_parameter_widths)[self.constrained_parameters]#reader.read_param(config_dict, 'constrained_widths', []) self.nuisance_parameters = reader.read_param(config_dict, 'nuisance_parameters', {}) - if 'res' in self.nuisance_parameters.keys(): - self.fix_res = not self.nuisance_parameters['res'] - self.constrained_parameters.append(6) - self.constrained_means.append(self.res_mean) - self.constrained_widths.append(self.res_width) - else: - self.fix_res = True - if 'scatter_peak_ratio_b' in self.nuisance_parameters.keys(): - self.fix_scatter_peak_ratio_b = not self.nuisance_parameters['scatter_peak_ratio_b'] - self.constrained_parameters.append(4) - self.constrained_means.append(self.scatter_peak_ratio_b_mean) - self.constrained_widths.append(self.scatter_peak_ratio_b_width) - else: - self.fix_scatter_peak_ratio_b = True + for i, p in enumerate(self.model_parameter_names): + if p in self.nuisance_parameters.keys(): + if self.nuisance_parameters[p]: + self.constrained_parameters.append(i) + self.constrained_means.append(self.model_parameter_means[i]) + self.constrained_widths.append(self.model_parameter_means[i]) + self.fixed_parameters[i] = False + else: + self.fixed_parameters[i] = True - if 'scatter_peak_ratio_c' in self.nuisance_parameters.keys(): - self.fix_scatter_peak_ratio_c = not self.nuisance_parameters['scatter_peak_ratio_c'] - self.constrained_parameters.append(5) - self.constrained_means.append(self.scatter_peak_ratio_c_mean) - self.constrained_widths.append(self.scatter_peak_ratio_c_width) - else: - self.fix_scatter_peak_ratio_c = True + # if 'res' in self.nuisance_parameters.keys(): + # self.fix_res = not self.nuisance_parameters['res'] + # self.constrained_parameters.append(6) + # self.constrained_means.append(self.res_mean) + # self.constrained_widths.append(self.res_width) + # else: + # self.fix_res = True - if 'two_gaussian_sigma_1' in self.nuisance_parameters.keys(): - self.fix_two_gaussian_sigma_1 = not self.nuisance_parameters['two_gaussian_sigma_1'] - self.constrained_parameters.append(7) - self.constrained_means.append(self.two_gaussian_sigma_1_mean) - self.constrained_widths.append(self.two_gaussian_sigma_1_width) - else: - self.fix_two_gaussian_sigma_1 = True + # if 'scatter_peak_ratio_b' in self.nuisance_parameters.keys(): + # self.fix_scatter_peak_ratio_b = not self.nuisance_parameters['scatter_peak_ratio_b'] + # self.constrained_parameters.append(4) + # self.constrained_means.append(self.scatter_peak_ratio_b_mean) + # self.constrained_widths.append(self.scatter_peak_ratio_b_width) + # else: + # self.fix_scatter_peak_ratio_b = True - if 'two_gaussian_sigma_2' in self.nuisance_parameters.keys(): - self.fix_two_gaussian_sigma_2 = not self.nuisance_parameters['two_gaussian_sigma_2'] - self.constrained_parameters.append(8) - self.constrained_means.append(self.two_gaussian_sigma_2_mean) - self.constrained_widths.append(self.two_gaussian_sigma_2_width) - else: - self.fix_two_gaussian_sigma_2 = True + # if 'scatter_peak_ratio_c' in self.nuisance_parameters.keys(): + # self.fix_scatter_peak_ratio_c = not self.nuisance_parameters['scatter_peak_ratio_c'] + # self.constrained_parameters.append(5) + # self.constrained_means.append(self.scatter_peak_ratio_c_mean) + # self.constrained_widths.append(self.scatter_peak_ratio_c_width) + # else: + # self.fix_scatter_peak_ratio_c = True - """if 'B' in self.nuisance_parameters.keys(): - self.fix_B = not self.nuisance_parameters['B'] - self.constrained_parameters.append(9) - self.constrained_means.append(self.B_mean) - self.constrained_widths.append(self.B_width)""" + # if 'two_gaussian_sigma_1' in self.nuisance_parameters.keys(): + # self.fix_two_gaussian_sigma_1 = not self.nuisance_parameters['two_gaussian_sigma_1'] + # self.constrained_parameters.append(7) + # self.constrained_means.append(self.two_gaussian_sigma_1_mean) + # self.constrained_widths.append(self.two_gaussian_sigma_1_width) + # else: + # self.fix_two_gaussian_sigma_1 = True + + # if 'two_gaussian_sigma_2' in self.nuisance_parameters.keys(): + # self.fix_two_gaussian_sigma_2 = not self.nuisance_parameters['two_gaussian_sigma_2'] + # self.constrained_parameters.append(8) + # self.constrained_means.append(self.two_gaussian_sigma_2_mean) + # self.constrained_widths.append(self.two_gaussian_sigma_2_width) + # else: + # self.fix_two_gaussian_sigma_2 = True + + # """if 'B' in self.nuisance_parameters.keys(): + # self.fix_B = not self.nuisance_parameters['B'] + # self.constrained_parameters.append(9) + # self.constrained_means.append(self.B_mean) + # self.constrained_widths.append(self.B_width)""" if len(self.constrained_parameters) > 0: logger.warning('Some parameters are constrained: {} - {}'.format(self.constrained_parameters, self.constrained_parameter_names)) @@ -353,7 +442,7 @@ def InternalConfigure(self, config_dict): raise ValueError('Max frequency undetermined') # ================================== - # lineshape details + # configure lineshape # ================================== # simplified lineshape parameters @@ -419,30 +508,6 @@ def InternalConfigure(self, config_dict): - # ================================== - # initial values - # ================================== - - self.B = self.B_mean - self.res = self.res_mean - if self.res <= 30.01/float(2*np.sqrt(2*np.log(2))): - logger.warning('Resolution small for shallow trap model. Setting to {}'.format(30.01/float(2*np.sqrt(2*np.log(2))))) - self.res = 30.01/float(2*np.sqrt(2*np.log(2))) - - self.endpoint=reader.read_param(config_dict, 'true_endpoint', 18.573e3) - self.background=reader.read_param(config_dict, 'initial_background', 0) - self.two_gaussian_sigma_1 = self.two_gaussian_sigma_1_mean - self.two_gaussian_sigma_2 = self.two_gaussian_sigma_2_mean - self.width_scaling = self.scale_mean - - #if self.use_fixed_scatter_peak_ratio: - # self.scatter_peak_ratio_b = self.scatter_peak_ratio_mean - # self.scatter_peak_ratio_c = 1 - #else: - self.scatter_peak_ratio_b = self.scatter_peak_ratio_b_mean - self.scatter_peak_ratio_c = self.scatter_peak_ratio_c_mean - self.parameter_samples = {} - # ================================== # energies and bins @@ -501,37 +566,37 @@ def ConfigureFit(self): # configure fit # this should not be hard coded - neutrino_limits = [-400**2, 400**2] - energy_limits = [max([self.endpoint-1000, np.min(self.energies)+np.sqrt(neutrino_limits[1])]), min([self.endpoint+1000, np.max(self.energies)-np.sqrt(np.abs(neutrino_limits[0]))])] - if self.print_level == 1: - logger.warning('Neutrino mass limited to: {} - {}'.format(*np.sqrt(np.array(neutrino_limits)))) + #neutrino_limits = [-400**2, 400**2] + #energy_limits = [max([self.endpoint-1000, np.min(self.energies)+np.sqrt(neutrino_limits[1])]), min([self.endpoint+1000, np.max(self.energies)-np.sqrt(np.abs(neutrino_limits[0]))])] + #if self.print_level == 1: + # logger.warning('Neutrino mass limited to: {} - {}'.format(*np.sqrt(np.array(neutrino_limits)))) #if not self.fit_efficiency_tilt: - self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', - 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', - 'res', 'two_gaussia_sigma_1', 'two_gaussian_sigma_2'] - self.initial_values = [self.endpoint, self.background, self.mass_guess**2, self.counts_guess, - self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, - self.res, self.two_gaussian_sigma_1, self.two_gaussian_sigma_2] + self.parameter_names = self.model_parameter_names #['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', + # 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', + # 'res', 'two_gaussia_sigma_1', 'two_gaussian_sigma_2'] + self.initial_values = self.model_parameter_means#[self.endpoint, self.background, self.mass_guess**2, self.counts_guess, + #self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, + #self.res, self.two_gaussian_sigma_1, self.two_gaussian_sigma_2] - self.fixes = [self.fix_endpoint, self.fix_background, self.fix_nu_mass, self.fix_amplitude, - self.fix_scatter_peak_ratio_b, self.fix_scatter_peak_ratio_c, - self.fix_res, self.fix_two_gaussian_sigma_1, self.fix_two_gaussian_sigma_2] + self.fixes = self.fixed_parameters #[self.fix_endpoint, self.fix_background, self.fix_nu_mass, self.fix_amplitude, + #self.fix_scatter_peak_ratio_b, self.fix_scatter_peak_ratio_c, + #self.fix_res, self.fix_two_gaussian_sigma_1, self.fix_two_gaussian_sigma_2] - self.limits = [energy_limits, - [0, None], - neutrino_limits, - [100, None], - [0.1, 1.], - [0.1, 1.], - [30, 100], - [1, 100], - [1, 100]] + # self.limits = [energy_limits, + # [0, None], + # neutrino_limits, + # [100, None], + # [0.1, 1.], + # [0.1, 1.], + # [30, 100], + # [1, 100], + # [1, 100]] - self.parameter_errors = [max([0.1, 0.1*p]) for p in self.initial_values] + self.parameter_errors = [max([0.1, p]) for p in self.model_parameter_widths] # else: @@ -625,6 +690,10 @@ def SamplePriors(self, sampled_parameters): if k in self.nuisance_parameters and self.nuisance_parameters[k]: raise ValueError('{} is nuisance parameter.'.format(k)) + for p in self.model_parameter_names: + if p in sampled_parameters.keys() and sampled_parameters[p]: + raise ValueError('{} is a free parameter'.format(p)) + logger.info('Sampling: {}'.format([k for k in sampled_parameters.keys() if sampled_parameters[k]])) self.parameter_samples = {} sample_values = [] @@ -685,7 +754,7 @@ def SamplePriors(self, sampled_parameters): self.parameter_samples['B'] = self.B sample_values.append(self.B) if 'endpoint' in sampled_parameters.keys() and sampled_parameters['endpoint']: - self.endpoint = self.Gaussian_sample(self.endpoint_mean, self.endpoint_width) + self.model_parameter_means[self.endpoint_index] = self.Gaussian_sample(self.endpoint_mean, self.endpoint_width) self.parameter_samples['endpoint'] = self.endpoint self.fix_endpoint = True sample_values.append(self.endpoint) @@ -1124,15 +1193,37 @@ def running_mean(self, x, N): return (cumsum[int(N_round)::int(N_round)] - cumsum[:-int(N_round):int(N_round)]) / float(N_round) - def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=None, res=None, sig1=None, sig2=None, tilt=0., error=False): + def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=None, res=None, sig1=None, sig2=None, tilt=0., error=False): E = np.array(E) - if sig1 == None: - sig1 = self.two_gaussian_sigma_1 - if sig2 == None: - sig2 = self.two_gaussian_sigma_2 - if res == None: - res = self.res + + # lineshape params + if self.is_scattered: + prob_b = args[self.scatter_peak_ratio_b_index] + prob_c = args[self.scatter_peak_ratio_c_index] + + if self.fixed_parameters[self.scatter_peak_ratio_b_index]: + prob_b = self.scatter_peak_ratio_b + if self.fixed_parameters[self.scatter_peak_ratio_c_index]: + prob_c = self.scatter_peak_ratio_c + + if self.is_smeared: + res = args[self.res_index] + if self.fixed_parameters[self.res_index]: + res = self.res + + if self.resolution_model != 'gaussian': + sig1 = args[self.two_gaussian_sigma_1_index] + sig2 = args[self.two_gaussian_sigma_2_index] + + if self.fixed_parameters[self.two_gaussian_sigma_1_index]: + sig1 = self.two_gaussian_sigma_1 + if self.fixed_parameters[self.two_gaussian_sigma_1_index]: + sig2 = self.two_gaussian_sigma_2 + + # tritium args + tritium_args = np.array(args)[self.tritium_model_indices] + if len(E)==0: E = self._bin_centers @@ -1173,8 +1264,8 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No lineshape = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, sig1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, sig2*self.width_scaling, self.two_gaussian_mu_2) # spectrum shape - spec = self.which_model(e_spec, endpoint, m_nu) - spec[np.where(e_spec>endpoint-np.abs(m_nu)**0.5)]=0. + spec = self.which_model(e_spec, *tritium_args)#endpoint, m_nu) + #spec[np.where(e_spec>args[self.endpoint_index]-np.abs(m_nu)**0.5)]=0. if not self.is_scattered: # convolve with gauss with spectrum @@ -1238,7 +1329,7 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No if self.plot_lineshape: logger.info('Using Gaussian resolution model') else: - resolution = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sigma_1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, self.two_gaussian_sigma_2*self.width_scaling, self.two_gaussian_mu_2) + resolution = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, sig1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, sig2*self.width_scaling, self.two_gaussian_mu_2) if self.plot_lineshape: logger.info('Using two Gaussian resolution model') @@ -1251,10 +1342,10 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No if FWHM < 30.01: logger.warning('FWHM smaller 30. Setting to 30 instead.') FWHM = 30.01 - logger.info('Plotting lineshape for FWHM {}, probs {} and {} and hydrogen proportion {}.'.format(FWHM, prob_b, prob_c, self.hydrogen_proportion)) - simple_ls, simple_norm = self.simplified_ls(e_lineshape, 0, FWHM, prob_b, prob_c) - simple_ls = (self.gauss_resolution_f(e_lineshape, 1, res, 0)+simple_ls)/simple_norm - plt.plot(e_lineshape, simple_ls/np.nanmax(simple_ls), label='Hydrogen only lineshape', color='red') + #logger.info('Plotting lineshape for FWHM {} and hydrogen proportion {}.'.format(FWHM, self.hydrogen_proportion)) + #simple_ls, simple_norm = self.simplified_ls(e_lineshape, 0, FWHM, prob_b, prob_c) + #simple_ls = (self.gauss_resolution_f(e_lineshape, 1, res, 0)+simple_ls)/simple_norm + #plt.plot(e_lineshape, simple_ls/np.nanmax(simple_ls), label='Hydrogen only lineshape', color='red') plt.xlabel('Energy [eV]') plt.ylabel('Amplitude') plt.grid() @@ -1267,22 +1358,10 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No else: # base shape - spec=np.zeros(len(self._energies)) - index=np.where(self._energies<=endpoint-np.abs(m_nu)**0.5) - try: - tritium = self.which_model(self.energies[index], endpoint, m_nu) - except Exception as e: - print(E) - print(index) - print(endpoint) - print(m_nu) - print(self.is_smeared) - raise(e) - - spec[index]=tritium + spec = self.which_model(e_spec, *tritium_args) K_convolved = spec - # integrate bins + # Fake integration if self.integrate_bins: dE = self.denergy @@ -1340,18 +1419,18 @@ def TritiumSpectrum(self, E=[], endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=No - def TritiumSpectrumBackground(self, E=[], endpoint=18.6e3, background=0, m_nu=0, amplitude=1., prob_b=None, prob_c=None, res=None, sig1=None, sig2=None, tilt=0., error=False): + def TritiumSpectrumBackground(self, E=[], *args): if len(E)==0: E = self.bin_centers - if error: - K, K_error = self.TritiumSpectrum(E, endpoint, m_nu, prob_b, prob_c, res, sig1, sig2, tilt, error) - K_norm = np.sum(self.TritiumSpectrum(self._energies, endpoint, m_nu, prob_b, prob_c, res, sig1, sig2, tilt, error=False)*(self._energies[1]-self._energies[0])) + if self.pass_efficiency_error: + K, K_error = self.TritiumSpectrum(E, *args, error=True)#endpoint, m_nu, prob_b, prob_c, res, sig1, sig2, tilt) + K_norm = np.sum(self.TritiumSpectrum(self._energies, *args))#endpoint, m_nu, prob_b, prob_c, res, sig1, sig2, tilt)*(self._energies[1]-self._energies[0])) else: - K = self.TritiumSpectrum(E, endpoint, m_nu, prob_b, prob_c, res, sig1, sig2, tilt, error) - K_norm = np.sum(self.TritiumSpectrum(self._energies, endpoint, m_nu, prob_b, prob_c, res, sig1, sig2, tilt, error=False)*(self._energies[1]-self._energies[0])) + K = self.TritiumSpectrum(E, *args)#endpoint, m_nu, prob_b, prob_c, res, sig1, sig2, tilt) + K_norm = np.sum(self.TritiumSpectrum(self._energies, *args))#endpoint, m_nu, prob_b, prob_c, res, sig1, sig2, tilt)*(self._energies[1]-self._energies[0])) @@ -1361,8 +1440,8 @@ def TritiumSpectrumBackground(self, E=[], endpoint=18.6e3, background=0, m_nu=0, return K*(E[1]-E[0])/K_norm else: - b = background/(max(self._energies)-min(self._energies)) - a = amplitude#-b + b = args[self.background_index]/(max(self._energies)-min(self._energies)) + a = args[self.amplitude_index]#-b B = np.ones(np.shape(E)) B = B*b*(E[1]-E[0])#/(self._energies[1]-self._energies[0]) @@ -1376,21 +1455,21 @@ def TritiumSpectrumBackground(self, E=[], endpoint=18.6e3, background=0, m_nu=0, - if error: + if self.pass_efficiency_error: K_error = K_error*(E[1]-E[0])/K_norm*a #K_error = K_error/np.sum(K)*a return K+B, K_error else: return K+B - def normalized_TritiumSpectrumBackground(self, E=[], endpoint=18.6e3, background=0, m_nu=0., amplitude=1., prob_b=None, prob_c=None, res=None, sig1=None, sig2=None, tilt=0., error=False): + def normalized_TritiumSpectrumBackground(self, E=[], *args):#endpoint=18.6e3, background=0, m_nu=0., amplitude=1., prob_b=None, prob_c=None, res=None, sig1=None, sig2=None, tilt=0., error=False): - if error: - t, t_error = self.TritiumSpectrumBackground(E, endpoint, background, m_nu, amplitude, prob_b, prob_c, res, sig1, sig2, tilt, error=error) + if self.pass_efficiency_error: + t, t_error = self.TritiumSpectrumBackground(E, *args)#endpoint, background, m_nu, amplitude, prob_b, prob_c, res, sig1, sig2, tilt) t_norm = np.sum(t) return t/t_norm, t_error/t_norm else: - t = self.TritiumSpectrumBackground(E, endpoint, background, m_nu, amplitude, prob_b, prob_c, res, sig1, sig2, tilt, error=error) + t = self.TritiumSpectrumBackground(E, *args)#endpoint, background, m_nu, amplitude, prob_b, prob_c, res, sig1, sig2, tilt) t_norm = np.sum(t) return t/t_norm From a17f87a750c2d978aa44ca7e5810f7c744f85f2c Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 14 Oct 2021 00:03:30 -0700 Subject: [PATCH 128/199] missed a comment --- mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 29d4bee1..0ec64f9e 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -204,7 +204,7 @@ def InternalConfigure(self, config_dict): self.parameter_samples = {} - + # need to know which parameter is background self.background_index = self.model_parameter_names.index('background') # and which one is B # self.B_index = self.model_parameter_names.index('B') From 58bc31d356949c83eeff5a45ef743e54a8cd765d Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 14 Oct 2021 00:16:03 -0700 Subject: [PATCH 129/199] fixes --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 0ec64f9e..34f8c9a7 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -330,8 +330,8 @@ def InternalConfigure(self, config_dict): # Parameters can be constrained manually or by inlcuding them in the nuisance parameter dictionary #self.constrained_parameter_names = reader.read_param(config_dict, 'constrained_parameter_names', []) self.constrained_parameters = reader.read_param(config_dict, 'constrained_parameters', []) - self.constrained_means = np.array(self.model_parameter_means)[self.constrained_parameters]#reader.read_param(config_dict, 'constrained_means', []) - self.constrained_widths = np.array(self.model_parameter_widths)[self.constrained_parameters]#reader.read_param(config_dict, 'constrained_widths', []) + self.constrained_means = list(np.array(self.model_parameter_means)[self.constrained_parameters])#reader.read_param(config_dict, 'constrained_means', []) + self.constrained_widths = list(np.array(self.model_parameter_widths)[self.constrained_parameters])#reader.read_param(config_dict, 'constrained_widths', []) self.nuisance_parameters = reader.read_param(config_dict, 'nuisance_parameters', {}) @@ -342,9 +342,8 @@ def InternalConfigure(self, config_dict): self.constrained_parameters.append(i) self.constrained_means.append(self.model_parameter_means[i]) self.constrained_widths.append(self.model_parameter_means[i]) - self.fixed_parameters[i] = False - else: - self.fixed_parameters[i] = True + if i in self.constrained_parameters: + self.fixed_parameters[i] = False # if 'res' in self.nuisance_parameters.keys(): # self.fix_res = not self.nuisance_parameters['res'] @@ -392,9 +391,6 @@ def InternalConfigure(self, config_dict): # self.constrained_means.append(self.B_mean) # self.constrained_widths.append(self.B_width)""" - if len(self.constrained_parameters) > 0: - logger.warning('Some parameters are constrained: {} - {}'.format(self.constrained_parameters, self.constrained_parameter_names)) - #self.print_level = 1 # MC uncertainty propagation does not need the fit uncertainties returned by iminuit. uncertainties are instead obtained from the distribution of fit results. # But if the uncertainty is unstead propagated by adding constraiend nuisance parameters then the fit uncertainties are needed. @@ -690,19 +686,19 @@ def SamplePriors(self, sampled_parameters): if k in self.nuisance_parameters and self.nuisance_parameters[k]: raise ValueError('{} is nuisance parameter.'.format(k)) - for p in self.model_parameter_names: - if p in sampled_parameters.keys() and sampled_parameters[p]: + for i, p in enumerate(self.model_parameter_names): + if p in sampled_parameters.keys() and sampled_parameters[p] and not self.fixed_parameters[i]: raise ValueError('{} is a free parameter'.format(p)) logger.info('Sampling: {}'.format([k for k in sampled_parameters.keys() if sampled_parameters[k]])) self.parameter_samples = {} sample_values = [] - if 'res' in sampled_parameters.keys() and sampled_parameters['res']: + if 'resolution' in sampled_parameters.keys() and sampled_parameters['resolution']: self.res = self.Gaussian_sample(self.res_mean, self.res_width) if self.res <= 30.01/float(2*np.sqrt(2*np.log(2))): logger.warning('Sampled resolution small. Setting to {}'.format(30.01/float(2*np.sqrt(2*np.log(2))))) self.res = 30.01/float(2*np.sqrt(2*np.log(2))) - self.parameter_samples['res'] = self.res + self.parameter_samples['resolution'] = self.res sample_values.append(self.res) if 'two_gaussian_sigma_1' in sampled_parameters.keys() and sampled_parameters['two_gaussian_sigma_1']: self.two_gaussian_sigma_1 = self.Gaussian_sample(self.two_gaussian_sigma_1_mean, self.two_gaussian_sigma_1_width) From e37359653e549d013409f0cfd15d4f9aedeb24d8 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Sun, 17 Oct 2021 14:27:04 -0400 Subject: [PATCH 130/199] Fix livetime correction --- .../TritiumSpectrum/FakeDataGenerator.py | 47 +++++++------------ 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 6f67b6c8..a23c2ea9 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -425,58 +425,47 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 ratesS[ratesS<0.] = 0. ratesB[ratesB<0.] = 0. - rate_sumS, rate_sumB = np.sum(ratesS), np.sum(ratesB) - probsS = np.array(ratesS)/rate_sumS - probsB = np.array(ratesB)/rate_sumB + #rate_sumS, rate_sumB = np.sum(ratesS), np.sum(ratesB) + #probsS = np.array(ratesS)/rate_sumS + #probsB = np.array(ratesB)/rate_sumB - #Calculate three different probs variables, for each of the three runtimes + #Calculate three different rates variables, for each of the three runtimes runtime_ratios = [t/float(self.channel_runtimes[0]) for t in self.channel_runtimes] logger.info('Generating data') time4 = time.time() #Break up self.Koptions into three different arrays. - #Then, sample KE variables for each of the arrays and appropriate elements of self.channel_runtimes and self.probs. + #Then, sample KE variables for each of the arrays and appropriate elements of self.channel_runtimes, ratesS and ratesB. #Finally, concatenate together the three KE arrays. - temp_Koptions, temp_probsS, temp_probsB = self.Koptions, probsS, probsB - split_Koptions, split_probsS, split_probsB = [], [], [] + temp_Koptions, temp_ratesS, temp_ratesB = self.Koptions, ratesS, ratesB + split_Koptions, split_ratesS, split_ratesB = [], [], [] for i in range(len(self.channel_bounds)): - print(len(temp_Koptions), len(temp_probsS), len(temp_probsB)) + print(len(temp_Koptions), len(temp_ratesS), len(temp_ratesB)) split_Koptions.append(temp_Koptions[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) - split_probsS.append(temp_probsS[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) - split_probsB.append(temp_probsB[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) - temp_probsS = temp_probsS[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] - temp_probsB = temp_probsB[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] + split_ratesS.append(temp_ratesS[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) + split_ratesB.append(temp_ratesB[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) + temp_ratesS = temp_ratesS[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] + temp_ratesB = temp_ratesB[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] temp_Koptions = temp_Koptions[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] split_Koptions.append(temp_Koptions) - split_probsS.append(temp_probsS) - split_probsB.append(temp_probsB) + split_ratesS.append(temp_ratesS) + split_ratesB.append(temp_ratesB) - self.probs = [] + rates = [] for i in range(len(self.channel_runtimes)): - self.probs.append((S*runtime_ratios[i]*split_probsS[i] + B*split_probsB[i])/(S*runtime_ratios[i]+B)) - - print(len(split_Koptions[0])) - print(len(self.probs[0])) + rates.append((S*runtime_ratios[i]*split_ratesS[i] + B*split_ratesB[i])/(S*runtime_ratios[i]+B)) self.Koptions = np.concatenate(split_Koptions) - self.probs = np.concatenate(self.probs) + rates = np.concatenate(rates) + self.probs = rates/np.sum(rates) if self.poisson_stats: KE = np.random.choice(self.Koptions, np.random.poisson(S+B), p = self.probs) else: KE = np.random.choice(self.Koptions, round(S+B), p = self.probs) - """ - split_KE = [] - for i in range(len(runtime_ratios)): - if self.poisson_stats: - split_KE.append(np.random.choice(split_Koptions[i], np.random.poisson(S*runtime_ratios[i]+B), p = self.probs[i])) - else: - split_KE.append(np.random.choice(split_Koptions[i], round(S*runtime_ratios[i]+B), p = self.probs[i])) - """ - time5 = time.time() logger.info('... took {} s'.format(time5-time4)) From c738edb36dff95cd7481e1697d613fbdd1b15b31 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Mon, 18 Oct 2021 13:19:48 -0700 Subject: [PATCH 131/199] moving param reads and if statements to where they are needed --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 157 +++++++++--------- 1 file changed, 79 insertions(+), 78 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 34f8c9a7..82fcd8eb 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -15,24 +15,22 @@ """ -import numpy as np -from scipy import constants -from scipy.special import erfc import json import os from copy import deepcopy +import numpy as np +from scipy import constants +from scipy.special import erfc import matplotlib.pyplot as plt -from mermithid.processors.Fitters import BinnedDataFitter -from mermithid.misc.FakeTritiumDataFunctions import * -from mermithid.processors.misc.KrComplexLineShape import KrComplexLineShape - from morpho.utilities import morphologging, reader logger = morphologging.getLogger(__name__) - -from mermithid.misc.DetectionEfficiencyUtilities import pseudo_integrated_efficiency, integrated_efficiency +from mermithid.processors.Fitters import BinnedDataFitter +from mermithid.misc.FakeTritiumDataFunctions import * +from mermithid.processors.misc.KrComplexLineShape import KrComplexLineShape +from mermithid.misc.DetectionEfficiencyUtilities import pseudo_integrated_efficiency, integrated_efficiency, power_efficiency #electron_mass = constants.electron_mass/constants.e*constants.c**2 @@ -57,7 +55,6 @@ def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, logger.info('Sampling: {}'.format(i)) - T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) #T.error_scaling = error_scaling @@ -120,6 +117,7 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, else: return results, errors, total_counts + def GetPDF(fit_config_dict, params, plot=False): logger.info('Plotting lineshape: {}'.format(plot)) logger.info('PDF for params: {}'.format(params)) @@ -148,7 +146,7 @@ class BinnedTritiumMLFitter(BinnedDataFitter): def InternalConfigure(self, config_dict): # ==================================== - # Model options and parameter settings + # Model configuration # ==================================== self.use_approx_model = reader.read_param(config_dict, 'use_approximate_model', True) self.use_toy_model_efficiency = reader.read_param(config_dict, 'use_toy_model_efficiency', False) @@ -174,7 +172,7 @@ def InternalConfigure(self, config_dict): # configure model parameter names self.model_parameter_names = reader.read_param(config_dict, 'model_parameter_names', ['endpoint', 'background', 'm_beta_squared', 'Amplitude', - 'scatter_peak_ratio_p', 'scatter_peak_ratio_q', + 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', 'resolution', 'two_gaussian_sigma_1', 'two_gaussian_sigma_2'] ) # initial values and mean of constaints (if constraint) or mean of distribution (if sampled) self.model_parameter_means = reader.read_param(config_dict, 'model_parameter_means', [18.6e3, 0, 0, 5000, 0.8, 1, 15, 10, 10]) @@ -188,7 +186,7 @@ def InternalConfigure(self, config_dict): [100, None], [0.1, 1.], [0.1, 1.], - [30, 100], + [12, 100], [1, 100], [1, 100]]) @@ -199,7 +197,7 @@ def InternalConfigure(self, config_dict): # ==================================== - # Identify and configure parameters + # Parameter configuration # ==================================== self.parameter_samples = {} @@ -212,10 +210,43 @@ def InternalConfigure(self, config_dict): self.amplitude_index = self.model_parameter_names.index('Amplitude') + # resolutions + + self.scale_mean = reader.read_param(config_dict, 'scale_mean', 1) + self.scale_width = reader.read_param(config_dict, 'scale_width', 0) + self.width_scaling = self.scale_mean + + if self.is_smeared: + self.res_index = self.model_parameter_names.index('resolution') + + self.res_mean = reader.read_param(config_dict, 'gaussian_resolution_mean', 15.0) + self.res_width = reader.read_param(config_dict, 'gaussian_resolution_width', 1.0) + self.res = self.res_mean + if self.res <= 30.01/float(2*np.sqrt(2*np.log(2))): + logger.warning('Resolution small for shallow trap model. Setting to {}'.format(30.01/float(2*np.sqrt(2*np.log(2))))) + self.res = 30.01/float(2*np.sqrt(2*np.log(2))) + + + + if self.resolution_model != 'gaussian': + self.two_gaussian_sigma_1_index = self.model_parameter_names.index('two_gaussian_sigma_1') + self.two_gaussian_sigma_2_index = self.model_parameter_names.index('two_gaussian_sigma_2') + + self.two_gaussian_mu_1 = reader.read_param(config_dict, 'two_gaussian_mu1', 0) + self.two_gaussian_mu_2 = reader.read_param(config_dict, 'two_gaussian_mu2', 0) + self.two_gaussian_sigma_1_mean = reader.read_param(config_dict, 'two_gaussian_sigma_1_mean', 15) + self.two_gaussian_sigma_2_mean = reader.read_param(config_dict, 'two_gaussian_sigma_2_mean', 5) + self.two_gaussian_sigma_1_width = reader.read_param(config_dict, 'two_gaussian_sigma_1_width', 1) + self.two_gaussian_sigma_2_width = reader.read_param(config_dict, 'two_gaussian_sigma_2_width', 1) + self.two_gaussian_wide_fraction = reader.read_param(config_dict, 'two_gaussian_wide_fraction', 1.) + self.two_gaussian_sigma_1 = deepcopy(self.two_gaussian_sigma_1_mean) + self.two_gaussian_sigma_2 = deepcopy(self.two_gaussian_sigma_2_mean) + + # scatter peak ratio if self.is_scattered: - self.scatter_peak_ratio_b_index = self.model_parameter_names.index('scatter_peak_ratio_p') - self.scatter_peak_ratio_c_index = self.model_parameter_names.index('scatter_peak_ratio_q') + self.scatter_peak_ratio_b_index = self.model_parameter_names.index('scatter_peak_ratio_b') + self.scatter_peak_ratio_c_index = self.model_parameter_names.index('scatter_peak_ratio_c') self.scatter_peak_ratio_b_mean = reader.read_param(config_dict, 'scatter_peak_ratio_b_mean', 0.7) self.scatter_peak_ratio_b_width = reader.read_param(config_dict, 'scatter_peak_ratio_b_width', 0.1) @@ -240,34 +271,7 @@ def InternalConfigure(self, config_dict): [self.b_scale_corr*stds[2]*stds[0], self.c_scale_corr*stds[2]*stds[1], stds[2]**2]] - # resolutions - if self.is_smeared: - self.res_index = self.model_parameter_names.index('resolution') - - self.res_mean = reader.read_param(config_dict, 'gaussian_resolution_mean', 15.0) - self.res_width = reader.read_param(config_dict, 'gaussian_resolution_width', 1.0) - self.res = self.res_mean - if self.res <= 30.01/float(2*np.sqrt(2*np.log(2))): - logger.warning('Resolution small for shallow trap model. Setting to {}'.format(30.01/float(2*np.sqrt(2*np.log(2))))) - self.res = 30.01/float(2*np.sqrt(2*np.log(2))) - - self.scale_mean = reader.read_param(config_dict, 'scale_mean', 1) - self.scale_width = reader.read_param(config_dict, 'scale_width', 0) - self.width_scaling = self.scale_mean - - if self.resolution_model != 'gaussian': - self.two_gaussian_sigma_1_index = self.model_parameter_names.index('two_gaussian_sigma_1') - self.two_gaussian_sigma_2_index = self.model_parameter_names.index('two_gaussian_sigma_2') - self.two_gaussian_mu_1 = reader.read_param(config_dict, 'two_gaussian_mu1', 0) - self.two_gaussian_mu_2 = reader.read_param(config_dict, 'two_gaussian_mu2', 0) - self.two_gaussian_sigma_1_mean = reader.read_param(config_dict, 'two_gaussian_sigma_1_mean', 15) - self.two_gaussian_sigma_2_mean = reader.read_param(config_dict, 'two_gaussian_sigma_2_mean', 5) - self.two_gaussian_sigma_1_width = reader.read_param(config_dict, 'two_gaussian_sigma_1_width', 1) - self.two_gaussian_sigma_2_width = reader.read_param(config_dict, 'two_gaussian_sigma_2_width', 1) - self.two_gaussian_wide_fraction = reader.read_param(config_dict, 'two_gaussian_wide_fraction', 1.) - self.two_gaussian_sigma_1 = self.two_gaussian_sigma_1_mean - self.two_gaussian_sigma_2 = self.two_gaussian_sigma_2_mean # identify tritium parameters self.tritium_model_indices = reader.read_param(config_dict, 'tritium_model_parameters', [0, 2]) @@ -275,6 +279,7 @@ def InternalConfigure(self, config_dict): self.endpoint_index = self.model_parameter_names.index('endpoint') self.fixed_parameters[self.m_beta_index] = not self.fit_nu_mass self.endpoint=reader.read_param(config_dict, 'true_endpoint', 18.573e3) + logger.info('Tritium model parameters: {}'.format(np.array(self.model_parameter_names)[self.tritium_model_indices])) # final state spectrum @@ -306,12 +311,8 @@ def InternalConfigure(self, config_dict): - - - - # ================= - # fit configuration + # Fit configuration # ================= self.print_level = reader.read_param(config_dict, 'print_level', 0) @@ -402,7 +403,7 @@ def InternalConfigure(self, config_dict): # ================================== - # detection efficiency details + # detection efficiency configuration # ================================== self.tilted_efficiency = False @@ -438,7 +439,7 @@ def InternalConfigure(self, config_dict): raise ValueError('Max frequency undetermined') # ================================== - # configure lineshape + # lineshape configuration # ================================== # simplified lineshape parameters @@ -683,7 +684,7 @@ def Gamma_sample(self, mean, width): def SamplePriors(self, sampled_parameters): for k in sampled_parameters.keys(): - if k in self.nuisance_parameters and self.nuisance_parameters[k]: + if k in self.nuisance_parameters and self.nuisance_parameters[k] and sampled_parameters[k]: raise ValueError('{} is nuisance parameter.'.format(k)) for i, p in enumerate(self.model_parameter_names): @@ -1193,30 +1194,6 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p E = np.array(E) - # lineshape params - if self.is_scattered: - prob_b = args[self.scatter_peak_ratio_b_index] - prob_c = args[self.scatter_peak_ratio_c_index] - - if self.fixed_parameters[self.scatter_peak_ratio_b_index]: - prob_b = self.scatter_peak_ratio_b - if self.fixed_parameters[self.scatter_peak_ratio_c_index]: - prob_c = self.scatter_peak_ratio_c - - if self.is_smeared: - res = args[self.res_index] - if self.fixed_parameters[self.res_index]: - res = self.res - - if self.resolution_model != 'gaussian': - sig1 = args[self.two_gaussian_sigma_1_index] - sig2 = args[self.two_gaussian_sigma_2_index] - - if self.fixed_parameters[self.two_gaussian_sigma_1_index]: - sig1 = self.two_gaussian_sigma_1 - if self.fixed_parameters[self.two_gaussian_sigma_1_index]: - sig2 = self.two_gaussian_sigma_2 - # tritium args tritium_args = np.array(args)[self.tritium_model_indices] @@ -1242,6 +1219,20 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p # smear spectrum if self.is_smeared or self.is_scattered: + # resolution params + res = args[self.res_index] + if self.fixed_parameters[self.res_index]: + res = self.res + + if self.resolution_model != 'gaussian': + sig1 = args[self.two_gaussian_sigma_1_index] + sig2 = args[self.two_gaussian_sigma_2_index] + + if self.fixed_parameters[self.two_gaussian_sigma_1_index]: + sig1 = self.two_gaussian_sigma_1 + if self.fixed_parameters[self.two_gaussian_sigma_1_index]: + sig2 = self.two_gaussian_sigma_2 + max_energy = 1000 dE = self.energies[1]-self.energies[0]#E[1]-E[0] @@ -1273,6 +1264,16 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p if self.is_scattered: + # scatter params + prob_b = args[self.scatter_peak_ratio_b_index] + prob_c = args[self.scatter_peak_ratio_c_index] + + if self.fixed_parameters[self.scatter_peak_ratio_b_index]: + prob_b = self.scatter_peak_ratio_b + if self.fixed_parameters[self.scatter_peak_ratio_c_index]: + prob_c = self.scatter_peak_ratio_c + + if prob_b == None: prob_b = self.scatter_peak_ratio_b_mean prob_c = self.scatter_peak_ratio_c_mean @@ -1375,7 +1376,7 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p if E[0] -0.5*dE_bins >= self._energies[1]: K_convolved_cut = K_convolved[self.energies>=min(E[0] -0.5*dE_bins)] K_convolved = self.running_mean(K_convolved_cut, N) - logger.warning('Cutting spectrum below Kmin:{} > {}'.format(E[0]-0.5*dE_bins, self_energies[0])) + logger.warning('Cutting spectrum below Kmin:{} > {}'.format(E[0]-0.5*dE_bins, self._energies[0])) else: K_convolved = self.running_mean(K_convolved, N) @@ -1396,8 +1397,8 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p if self.is_distorted == True: - if self.fit_efficiency_tilt: - self.tilt = tilt + #if self.fit_efficiency_tilt: + # self.tilt = tilt efficiency, efficiency_errors = self.Efficiency(E) K_eff=K_convolved*efficiency From 10a124a873991d081e3c28622d62d783322804c7 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Tue, 19 Oct 2021 11:48:29 -0400 Subject: [PATCH 132/199] Fixed normalization process --- .../TritiumSpectrum/FakeDataGenerator.py | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index a23c2ea9..dc54fa32 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -2,7 +2,7 @@ Generate binned or pseudo unbinned data Author: T. Weiss, C. Claessens, X. Huyan Date: 4/6/2020 -Updated: 2/9/2021 +Updated: 10/19/2021 ''' from __future__ import absolute_import @@ -425,9 +425,9 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 ratesS[ratesS<0.] = 0. ratesB[ratesB<0.] = 0. - #rate_sumS, rate_sumB = np.sum(ratesS), np.sum(ratesB) - #probsS = np.array(ratesS)/rate_sumS - #probsB = np.array(ratesB)/rate_sumB + rate_sumS, rate_sumB = np.sum(ratesS), np.sum(ratesB) + probsS = np.array(ratesS)/rate_sumS + probsB = np.array(ratesB)/rate_sumB #Calculate three different rates variables, for each of the three runtimes runtime_ratios = [t/float(self.channel_runtimes[0]) for t in self.channel_runtimes] @@ -436,26 +436,25 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 time4 = time.time() #Break up self.Koptions into three different arrays. - #Then, sample KE variables for each of the arrays and appropriate elements of self.channel_runtimes, ratesS and ratesB. + #Then, sample KE variables for each of the arrays and appropriate elements of self.channel_runtimes, probsS and probsB. #Finally, concatenate together the three KE arrays. - temp_Koptions, temp_ratesS, temp_ratesB = self.Koptions, ratesS, ratesB - split_Koptions, split_ratesS, split_ratesB = [], [], [] + temp_Koptions, temp_probsS, temp_probsB = self.Koptions, probsS, probsB + split_Koptions, split_probsS, split_probsB = [], [], [] for i in range(len(self.channel_bounds)): - print(len(temp_Koptions), len(temp_ratesS), len(temp_ratesB)) split_Koptions.append(temp_Koptions[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) - split_ratesS.append(temp_ratesS[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) - split_ratesB.append(temp_ratesB[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) - temp_ratesS = temp_ratesS[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] - temp_ratesB = temp_ratesB[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] + split_probsS.append(temp_probsS[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) + split_probsB.append(temp_probsB[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) + temp_probsS = temp_probsS[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] + temp_probsB = temp_probsB[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] temp_Koptions = temp_Koptions[Frequency(temp_Koptions, self.B_field)>self.channel_bounds[i]] split_Koptions.append(temp_Koptions) - split_ratesS.append(temp_ratesS) - split_ratesB.append(temp_ratesB) + split_probsS.append(temp_probsS) + split_probsB.append(temp_probsB) rates = [] for i in range(len(self.channel_runtimes)): - rates.append((S*runtime_ratios[i]*split_ratesS[i] + B*split_ratesB[i])/(S*runtime_ratios[i]+B)) + rates.append((S*runtime_ratios[i]*split_probsS[i] + B*split_probsB[i])/(S*runtime_ratios[i]+B)) self.Koptions = np.concatenate(split_Koptions) rates = np.concatenate(rates) From f0909484b580d865959907febca24a7714727837 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 20 Oct 2021 09:55:37 -0400 Subject: [PATCH 133/199] update MultiGasComplexLineShape.py --- .../misc/MultiGasComplexLineShape.py | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index fa05181d..ecf76795 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -3503,3 +3503,211 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi return dictionary_of_fit_results + def generate_scatter_peaks(self, emitted_peak='shake'): + + p = np.zeros(len(self.gases)) + scatter_fraction = self.scatter_fractions_for_gases + p[0:-1] = scatter_fraction + p[-1] = 1 - sum(scatter_fraction) + + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + + scatter_peaks = np.zeros((self.max_scatters+1, len(en_array))) + if emitted_peak == 'lorentzian': + current_working_spectrum = self.std_lorenztian_17keV() + elif emitted_peak == 'shake': + current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() + elif emitted_peak == 'dirac': + current_working_spectrum = self.std_dirac() + + scale_factor = 1 + current_working_spectrum = self.convolve_simulated_resolution_scaled(current_working_spectrum, scale_factor) + zeroth_order_peak = current_working_spectrum + scatter_peaks[0] = zeroth_order_peak + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + current_scatter_peak_spectrum = np.zeros(len(en_array)) + gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) + for combination in gas_scatter_combinations: + entry_str = '' + for component, gas_type in zip(combination, self.gases): + entry_str += gas_type + entry_str += str(component).zfill(2) + current_working_spectrum = scatter_spectra.item()[entry_str] + current_working_spectrum = self.normalize(signal.convolve(zeroth_order_peak, current_working_spectrum, mode='same')) + coefficient = factorial(sum(combination)) + for component, i in zip(combination, range(N)): + coefficient = coefficient/factorial(component)*p[i]**component + current_scatter_peak_spectrum += coefficient*current_working_spectrum + scatter_peaks[M] = current_scatter_peak_spectrum + return scatter_peaks + + def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(self, scatter_peaks, survival_probability, scatter_peak_ratio_b, scatter_peak_ratio_c): + scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) + en_array = self.std_eV_array() + current_full_spectrum = np.zeros(len(en_array)) + current_full_spectrum += scatter_peaks[0] + N = len(self.gases) + for M in range(1, self.max_scatters + 1): + scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_b*M**( - self.factor*scatter_peak_ratio_b + scatter_peak_ratio_c))#(-0.5179*scatter_peak_ratio_b + scatter_peak_ratio_c) -0.448 -0.4934 + current_full_spectrum += scatter_peaks[M]*scatter_peak_ratio*survival_probability**M + return current_full_spectrum + + def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(self, bins_Hz, eff_array, scatter_peaks, *p0): + + B_field = p0[0] + amplitude = p0[1] + survival_probability = p0[2] + scatter_peak_ratio_b = p0[3] + scatter_peak_ratio_c = p0[4] + + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) + en_loss_array = self.std_eV_array() + en_loss_array_min = en_loss_array[0] + en_loss_array_max = en_loss_array[len(en_loss_array)-1] + f = np.zeros(len(x_eV)) + f_intermediate = np.zeros(len(x_eV)) + + x_eV_minus_line = Constants.kr_k_line_e() - x_eV + zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] + nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] + + full_spectrum = self.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(scatter_peaks, survival_probability, scatter_peak_ratio_b, scatter_peak_ratio_c) + f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) + f_intermediate = f_intermediate*eff_array + f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) + + return f + + def chi_2_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(self, bin_centers, data_hist_freq, eff_array, scatter_peaks, params): + # expectation + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(bin_centers, eff_array, scatter_peaks, *params) + nonzero_bins_index = np.where((data_hist_freq != 0) & (fit_Hz > 0)) + zero_bins_index = np.where((data_hist_freq == 0) | (fit_Hz <= 0)) + chi2 = 2*((fit_Hz - data_hist_freq + data_hist_freq*np.log(data_hist_freq/fit_Hz))[nonzero_bins_index]).sum() + chi2 += 2*(fit_Hz - data_hist_freq)[zero_bins_index].sum() + return chi2 + + def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(self, freq_bins, data_hist_freq, print_params=True): + t = time.time() + self.check_existence_of_scatter_file() + bins_Hz = freq_bins + self.RF_ROI_MIN + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + # Initial guesses for curve_fit + B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) + amplitude_guess = np.sum(data_hist_freq) + FWHM_eV_guess = 5 + survival_probability_guess = 0.5 + scatter_fraction_guess = 0.5 + sigma_guess = 5 + gamma_guess = 3 + gaussian_portion_guess = 0.5 + scale_factor_guess = 1 + scatter_peak_ratio_parameter_guess = 0.7 + # Bounds for curve_fit + B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) + B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) + amplitude_min = 1e-5 + amplitude_max = np.sum(data_hist_freq)*3 + FWHM_eV_min = 0 + FWHM_eV_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) + survival_probability_min = 1e-5 + survival_probability_max = 1 + scatter_fraction_min = 1e-5 + scatter_fraction_max = 1 + scale_factor_min = 1e-5 + scale_factor_max = 5 + scatter_peak_ratio_parameter_min = 1e-5 + scatter_peak_ratio_parameter_max = 5 + N = len(self.gases) + gas_scatter_fraction_parameter_str = [] + for i in range(N-1): + gas_scatter_fraction_parameter_str += [self.gases[i]+' scatter fraction'] + p0_guess = [B_field_guess, amplitude_guess, survival_probability_guess, scatter_peak_ratio_parameter_guess, scatter_peak_ratio_parameter_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min, amplitude_max), (survival_probability_min, survival_probability_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max)] + parameter_names = ['B field','amplitude', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] + + scatter_peaks = self.generate_scatter_peaks() + # Actually do the fitting + m_binned = Minuit(lambda p: self.chi_2_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(bins_Hz, data_hist_freq, eff_array, scatter_peaks, p), p0_guess, name = parameter_names) + m_binned.limits = p0_bounds + if len(self.fixed_parameter_names)>0: + for fixed_parameter_name, fixed_parameter_value in zip(self.fixed_parameter_names, self.fixed_parameter_values): + m_binned.fixed[fixed_parameter_name] = True + m_binned.values[fixed_parameter_name] = fixed_parameter_value + m_binned.errors[fixed_parameter_name] = 0 + m_binned.migrad() + m_binned.hesse() + params = m_binned.values[0:] + B_field_fit = params[0] + #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) + amplitude_fit = params[1] + survival_probability_fit = params[2] + scatter_peak_ratio_b_fit = params[3] + scatter_peak_ratio_c_fit = params[4] + total_counts_fit = amplitude_fit + logger.info('\n'+str(m_binned.params)) + + perr = m_binned.errors[0:] + B_field_fit_err = perr[0] + amplitude_fit_err = perr[1] + survival_probability_fit_err = perr[2] + scatter_peak_ratio_b_fit_err = perr[3] + scatter_peak_ratio_c_fit_err = perr[4] + total_counts_fit_err = amplitude_fit_err + + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(bins_Hz, eff_array, scatter_peaks, *params) + fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) + bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 + bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) + reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + correlation_matrix = m_binned.covariance.correlation() + + if print_params == True: + output_string = '\n' + output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) + output_string += '-----------------\n' + output_string += 'B field = {:.8e}'.format(B_field_fit)+' +/- '+ '{:.4e} T\n'.format(B_field_fit_err) + output_string += '-----------------\n' + output_string += 'Amplitude = {}'.format(round(amplitude_fit,2))+' +/- {}'.format(round(amplitude_fit_err,2)) + '\n' + output_string += '-----------------\n' + output_string += 'survival probability = {:.8e}'.format(survival_probability_fit) + ' +/- {:.8e}\n'.format(survival_probability_fit_err) + output_string += '-----------------\n' + output_string += 'scatter_peak_ratio_b = {:.8e}'.format(scatter_peak_ratio_b_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_b_fit_err) + output_string += '-----------------\n' + output_string += 'scatter_peak_ratio_c = {:.8e}'.format(scatter_peak_ratio_c_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_c_fit_err) + output_string += '-----------------\n' + scatter_fraction = np.zeros(len(self.gases)) + scatter_fraction[0:-1] = self.scatter_fractions_for_gases + scatter_fraction[-1] = 1 - sum(scatter_fraction) + for i in range(len(self.gases)): + output_string += '{} scatter fraction = '.format(self.gases[i]) + "{:.8e}".format(scatter_fraction[i])+'\n' + output_string += '-----------------\n' + elapsed = time.time() - t + output_string += 'Fit completed in '+str(round(elapsed,2))+'s'+'\n' + dictionary_of_fit_results = { + 'output_string': output_string, + 'perr': perr, + 'bins_keV': bins_keV, + 'fit_keV': fit_keV, + 'bins_Hz': bins_Hz, + 'fit_Hz': fit_Hz, + 'B_field_fit': B_field_fit, + 'B_field_fit_err': B_field_fit_err, + 'scatter_peak_ratio_b_fit': scatter_peak_ratio_b_fit, + 'scatter_peak_ratio_b_fit_err': scatter_peak_ratio_b_fit_err, + 'scatter_peak_ratio_c_fit': scatter_peak_ratio_c_fit, + 'scatter_peak_ratio_c_fit_err': scatter_peak_ratio_c_fit_err, + 'amplitude_fit': amplitude_fit, + 'amplitude_fit_err': amplitude_fit_err, + 'data_hist_freq': data_hist_freq, + 'reduced_chi2': reduced_chi2, + 'correlation_matrix': np.array(correlation_matrix) + } + return dictionary_of_fit_results \ No newline at end of file From 100948589d6b791a5f64118f7e6e7be3246c9bc9 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 20 Oct 2021 10:13:23 -0400 Subject: [PATCH 134/199] update MultiGasComplexLineShape.py --- mermithid/processors/misc/MultiGasComplexLineShape.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index ecf76795..4347c85e 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -102,6 +102,7 @@ def InternalConfigure(self, params): self.recon_eff_param_a = self.recon_eff_params[0] self.recon_eff_param_b = self.recon_eff_params[1] self.recon_eff_param_c = self.recon_eff_params[2] + self.factor = reader.read_param(params, 'factor', []) if not os.path.exists(self.shake_spectrum_parameters_json_path) and self.base_shape=='shake': raise IOError('Shake spectrum path does not exist') From c23ce65f8fa70137d383c9eda117b2de49a069b7 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 20 Oct 2021 10:21:48 -0400 Subject: [PATCH 135/199] notes for update: added fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor; added three configuration options: self.fix_gas_composition, self.fix_width_scale_factor, self.scatter_fractions_for_gases --- mermithid/processors/misc/MultiGasComplexLineShape.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 4347c85e..7ccde3d0 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -51,6 +51,9 @@ def InternalConfigure(self, params): # Read other parameters self.bins_choice = reader.read_param(params, 'bins_choice', []) self.gases = reader.read_param(params, 'gases', ["H2", "Kr", "He", "Ar"]) + self.fix_gas_composition = reader.read_param(params, 'fix_gas_composition', False) + self.fix_width_scale_factor = reader.read_param(params, 'fix_width_scale_factor', False) + self.scatter_fractions_for_gases = reader.read_param(params, 'scatter_fractions_for_gases', []) self.max_scatters = reader.read_param(params, 'max_scatters', 20) self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.003, 0.013, 0.014, 0.02]}) #Weights from Xueying's Sept. 13 slides; errors currently arbitrary self.fixed_scatter_proportion = reader.read_param(params, 'fixed_scatter_proportion', True) @@ -3504,7 +3507,7 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi return dictionary_of_fit_results - def generate_scatter_peaks(self, emitted_peak='shake'): + def generate_scatter_peaks(self, emitted_peak = self.base_shape): p = np.zeros(len(self.gases)) scatter_fraction = self.scatter_fractions_for_gases From 01dbb342930b2a59933f44e878395b824f353483 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 20 Oct 2021 10:24:54 -0400 Subject: [PATCH 136/199] update MultiGasComplexLineShape.py --- mermithid/processors/misc/MultiGasComplexLineShape.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 7ccde3d0..641e20e8 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -51,6 +51,9 @@ def InternalConfigure(self, params): # Read other parameters self.bins_choice = reader.read_param(params, 'bins_choice', []) self.gases = reader.read_param(params, 'gases', ["H2", "Kr", "He", "Ar"]) + # when self.fix_gas_composition and self.fix_width_scale_factor are both True, + # fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor is used, + # Then the first N-1 gas compositions are set below through self.scatter_fractions_for_gases self.fix_gas_composition = reader.read_param(params, 'fix_gas_composition', False) self.fix_width_scale_factor = reader.read_param(params, 'fix_width_scale_factor', False) self.scatter_fractions_for_gases = reader.read_param(params, 'scatter_fractions_for_gases', []) From 869bc03cc850014cd57656f56c8907713f9037aa Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 20 Oct 2021 11:16:06 -0400 Subject: [PATCH 137/199] update internal run --- mermithid/processors/misc/MultiGasComplexLineShape.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 641e20e8..1959c265 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -54,6 +54,7 @@ def InternalConfigure(self, params): # when self.fix_gas_composition and self.fix_width_scale_factor are both True, # fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor is used, # Then the first N-1 gas compositions are set below through self.scatter_fractions_for_gases + # Otherwise, fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio is used self.fix_gas_composition = reader.read_param(params, 'fix_gas_composition', False) self.fix_width_scale_factor = reader.read_param(params, 'fix_width_scale_factor', False) self.scatter_fractions_for_gases = reader.read_param(params, 'scatter_fractions_for_gases', []) @@ -166,10 +167,12 @@ def InternalRun(self): else: self.results = self.fit_data_simulated_resolution_scaled_fit_recon_eff(freq_bins, data_hist_freq) elif self.resolution_function == 'simulated_resolution_scaled_fit_scatter_peak_ratio': - self.results = self.fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(freq_bins, data_hist_freq) + if self.fix_gas_composition == True and self.fix_width_scale_factor == True: + self.results = self.fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(freq_bins, data_hist_freq) + else: + self.results = self.fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(freq_bins, data_hist_freq) elif self.resolution_function == 'gaussian_resolution_fit_scatter_peak_ratio': self.results = self.fit_data_gaussian_resolution_fit_scatter_peak_ratio(freq_bins, data_hist_freq) - return True From f43e4b63560a64b42204fc6d32e3564afb4ffc30 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 20 Oct 2021 14:27:02 -0400 Subject: [PATCH 138/199] update fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio --- .../misc/MultiGasComplexLineShape.py | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 1959c265..b21b9b58 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -3130,13 +3130,12 @@ def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() elif emitted_peak == 'dirac': current_working_spectrum = self.std_dirac() - current_working_spectrum = self.convolve_simulated_resolution_scaled(current_working_spectrum, scale_factor) zeroth_order_peak = current_working_spectrum current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_b*M**scatter_peak_ratio_c) + scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_b*M**( -self.factor*scatter_peak_ratio_b + scatter_peak_ratio_c))#(-0.5179*scatter_peak_ratio_b + scatter_peak_ratio_c) -0.448 gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: entry_str = '' @@ -3194,12 +3193,9 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - if self.use_quad_trap_eff_interp == True: - quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) - quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] - eff_array = quad_trap_count_rate_interp(bins_Hz) - else: - eff_array = np.ones(len(bins_Hz)) + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) # Initial guesses for curve_fit B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) amplitude_guess = np.sum(data_hist_freq) @@ -3209,8 +3205,9 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, sigma_guess = 5 gamma_guess = 3 gaussian_portion_guess = 0.5 - scale_factor_guess = 0.5 - scatter_peak_ratio_parameter_guess = 0.5 + scale_factor_guess = 1 + scatter_peak_ratio_parameter_b_guess = 0.9 + scatter_peak_ratio_parameter_c_guess = 1.0 # Bounds for curve_fit B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) @@ -3230,8 +3227,8 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, gas_scatter_fraction_parameter_str = [] for i in range(N-1): gas_scatter_fraction_parameter_str += [self.gases[i]+' scatter fraction'] - p0_guess = [B_field_guess, amplitude_guess, scale_factor_guess, survival_probability_guess, scatter_peak_ratio_parameter_guess, scatter_peak_ratio_parameter_guess]+ (N-1)*[scatter_fraction_guess] - p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (scale_factor_min, scale_factor_max), (survival_probability_min, survival_probability_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max)] + (N-1)*[(scatter_fraction_min, scatter_fraction_max)] + p0_guess = [B_field_guess, amplitude_guess, scale_factor_guess, survival_probability_guess, scatter_peak_ratio_parameter_b_guess, scatter_peak_ratio_parameter_c_guess]+ (N-1)*[scatter_fraction_guess] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min, amplitude_max), (scale_factor_min, scale_factor_max), (survival_probability_min, survival_probability_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max)] + (N-1)*[(scatter_fraction_min, scatter_fraction_max)] parameter_names = ['B field','amplitude','width scale factor', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] + gas_scatter_fraction_parameter_str # Actually do the fitting m_binned = Minuit(lambda p: self.chi_2_simulated_resolution_scaled_fit_scatter_peak_ratio(bins_Hz, data_hist_freq, eff_array, p), p0_guess, name = parameter_names) @@ -3270,6 +3267,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) + correlation_matrix = m_binned.covariance.correlation() if print_params == True: output_string = '\n' @@ -3311,7 +3309,8 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, 'data_hist_freq': data_hist_freq, - 'reduced_chi2': reduced_chi2 + 'reduced_chi2': reduced_chi2, + 'correlation_matrix': np.array(correlation_matrix) } return dictionary_of_fit_results @@ -3554,7 +3553,7 @@ def generate_scatter_peaks(self, emitted_peak = self.base_shape): scatter_peaks[M] = current_scatter_peak_spectrum return scatter_peaks - def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(self, scatter_peaks, survival_probability, scatter_peak_ratio_b, scatter_peak_ratio_c): + def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(self, scatter_peaks, survival_probability, scatter_peak_ratio_p, scatter_peak_ratio_q): scatter_spectra_file_path = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') scatter_spectra = np.load(scatter_spectra_file_path, allow_pickle = True) en_array = self.std_eV_array() @@ -3562,7 +3561,7 @@ def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_ current_full_spectrum += scatter_peaks[0] N = len(self.gases) for M in range(1, self.max_scatters + 1): - scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_b*M**( - self.factor*scatter_peak_ratio_b + scatter_peak_ratio_c))#(-0.5179*scatter_peak_ratio_b + scatter_peak_ratio_c) -0.448 -0.4934 + scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_p*M**( - self.factor*scatter_peak_ratio_p + scatter_peak_ratio_q))#(-0.5179*scatter_peak_ratio_b + scatter_peak_ratio_c) -0.448 -0.4934 current_full_spectrum += scatter_peaks[M]*scatter_peak_ratio*survival_probability**M return current_full_spectrum @@ -3641,7 +3640,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c gas_scatter_fraction_parameter_str += [self.gases[i]+' scatter fraction'] p0_guess = [B_field_guess, amplitude_guess, survival_probability_guess, scatter_peak_ratio_parameter_guess, scatter_peak_ratio_parameter_guess] p0_bounds = [(B_field_min,B_field_max), (amplitude_min, amplitude_max), (survival_probability_min, survival_probability_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max)] - parameter_names = ['B field','amplitude', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] + parameter_names = ['B field','amplitude', 'survival probability','scatter peak ratio param p', 'scatter peak ratio param q'] scatter_peaks = self.generate_scatter_peaks() # Actually do the fitting @@ -3659,8 +3658,8 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c #starting at index 2, grabs every other entry. (which is how scattering probs are filled in for N gases) amplitude_fit = params[1] survival_probability_fit = params[2] - scatter_peak_ratio_b_fit = params[3] - scatter_peak_ratio_c_fit = params[4] + scatter_peak_ratio_p_fit = params[3] + scatter_peak_ratio_q_fit = params[4] total_counts_fit = amplitude_fit logger.info('\n'+str(m_binned.params)) @@ -3668,8 +3667,8 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c B_field_fit_err = perr[0] amplitude_fit_err = perr[1] survival_probability_fit_err = perr[2] - scatter_peak_ratio_b_fit_err = perr[3] - scatter_peak_ratio_c_fit_err = perr[4] + scatter_peak_ratio_p_fit_err = perr[3] + scatter_peak_ratio_q_fit_err = perr[4] total_counts_fit_err = amplitude_fit_err fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(bins_Hz, eff_array, scatter_peaks, *params) @@ -3689,9 +3688,9 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c output_string += '-----------------\n' output_string += 'survival probability = {:.8e}'.format(survival_probability_fit) + ' +/- {:.8e}\n'.format(survival_probability_fit_err) output_string += '-----------------\n' - output_string += 'scatter_peak_ratio_b = {:.8e}'.format(scatter_peak_ratio_b_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_b_fit_err) + output_string += 'scatter_peak_ratio_p = {:.8e}'.format(scatter_peak_ratio_p_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_p_fit_err) output_string += '-----------------\n' - output_string += 'scatter_peak_ratio_c = {:.8e}'.format(scatter_peak_ratio_c_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_c_fit_err) + output_string += 'scatter_peak_ratio_q = {:.8e}'.format(scatter_peak_ratio_q_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_q_fit_err) output_string += '-----------------\n' scatter_fraction = np.zeros(len(self.gases)) scatter_fraction[0:-1] = self.scatter_fractions_for_gases @@ -3710,10 +3709,10 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c 'fit_Hz': fit_Hz, 'B_field_fit': B_field_fit, 'B_field_fit_err': B_field_fit_err, - 'scatter_peak_ratio_b_fit': scatter_peak_ratio_b_fit, - 'scatter_peak_ratio_b_fit_err': scatter_peak_ratio_b_fit_err, - 'scatter_peak_ratio_c_fit': scatter_peak_ratio_c_fit, - 'scatter_peak_ratio_c_fit_err': scatter_peak_ratio_c_fit_err, + 'scatter_peak_ratio_p_fit': scatter_peak_ratio_p_fit, + 'scatter_peak_ratio_p_fit_err': scatter_peak_ratio_p_fit_err, + 'scatter_peak_ratio_q_fit': scatter_peak_ratio_q_fit, + 'scatter_peak_ratio_q_fit_err': scatter_peak_ratio_q_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, 'data_hist_freq': data_hist_freq, From 57f45e985f0e4af1c3b9b1ba240bbcb578fd6e17 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 20 Oct 2021 14:34:06 -0400 Subject: [PATCH 139/199] update fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio --- mermithid/processors/misc/MultiGasComplexLineShape.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index b21b9b58..49976c02 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -3193,9 +3193,12 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) - quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] - eff_array = quad_trap_count_rate_interp(bins_Hz) + if self.use_quad_trap_eff_interp == True: + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + else: + eff_array = np.ones(len(bins_Hz)) # Initial guesses for curve_fit B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) amplitude_guess = np.sum(data_hist_freq) From 8011208d99d7ae87342415017a333e513a734800 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 20 Oct 2021 14:40:02 -0400 Subject: [PATCH 140/199] update fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio --- .../misc/MultiGasComplexLineShape.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 49976c02..941a0468 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -3116,7 +3116,7 @@ def fit_data_simulated_resolution_scaled_fit_recon_eff(self, freq_bins, data_his } return dictionary_of_fit_results - def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale_factor, survival_probability, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='shake'): + def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale_factor, survival_probability, scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, emitted_peak='shake'): p = np.zeros(len(self.gases)) p[0:-1] = scatter_fraction p[-1] = 1 - sum(scatter_fraction) @@ -3135,7 +3135,7 @@ def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_b*M**( -self.factor*scatter_peak_ratio_b + scatter_peak_ratio_c))#(-0.5179*scatter_peak_ratio_b + scatter_peak_ratio_c) -0.448 + scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_p*M**( -self.factor*scatter_peak_ratio_p + scatter_peak_ratio_q))#(-0.5179*scatter_peak_ratio_b + scatter_peak_ratio_c) -0.448 gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: entry_str = '' @@ -3172,7 +3172,7 @@ def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(self, bins_ zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - full_spectrum = self.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale_factor, survival_probability, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction) + full_spectrum = self.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale_factor, survival_probability, scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f_intermediate = f_intermediate*eff_array f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) @@ -3209,8 +3209,8 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, gamma_guess = 3 gaussian_portion_guess = 0.5 scale_factor_guess = 1 - scatter_peak_ratio_parameter_b_guess = 0.9 - scatter_peak_ratio_parameter_c_guess = 1.0 + scatter_peak_ratio_parameter_p_guess = 0.9 + scatter_peak_ratio_parameter_q_guess = 1.0 # Bounds for curve_fit B_field_min = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[0]) B_field_max = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[-1]) @@ -3230,7 +3230,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, gas_scatter_fraction_parameter_str = [] for i in range(N-1): gas_scatter_fraction_parameter_str += [self.gases[i]+' scatter fraction'] - p0_guess = [B_field_guess, amplitude_guess, scale_factor_guess, survival_probability_guess, scatter_peak_ratio_parameter_b_guess, scatter_peak_ratio_parameter_c_guess]+ (N-1)*[scatter_fraction_guess] + p0_guess = [B_field_guess, amplitude_guess, scale_factor_guess, survival_probability_guess, scatter_peak_ratio_parameter_p_guess, scatter_peak_ratio_parameter_q_guess]+ (N-1)*[scatter_fraction_guess] p0_bounds = [(B_field_min,B_field_max), (amplitude_min, amplitude_max), (scale_factor_min, scale_factor_max), (survival_probability_min, survival_probability_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max)] + (N-1)*[(scatter_fraction_min, scatter_fraction_max)] parameter_names = ['B field','amplitude','width scale factor', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] + gas_scatter_fraction_parameter_str # Actually do the fitting @@ -3249,8 +3249,8 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, amplitude_fit = params[1] scale_factor_fit = params[2] survival_probability_fit = params[3] - scatter_peak_ratio_b_fit = params[4] - scatter_peak_ratio_c_fit = params[5] + scatter_peak_ratio_p_fit = params[4] + scatter_peak_ratio_q_fit = params[5] total_counts_fit = amplitude_fit logger.info('\n'+str(m_binned.params)) scatter_fraction_fit = params[6:5+N]+[1- sum(params[6:5+N])] @@ -3260,8 +3260,8 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, amplitude_fit_err = perr[1] scale_factor_fit_err = perr[2] survival_probability_fit_err = perr[3] - scatter_peak_ratio_b_fit_err = perr[4] - scatter_peak_ratio_c_fit_err = perr[5] + scatter_peak_ratio_p_fit_err = perr[4] + scatter_peak_ratio_q_fit_err = perr[5] total_counts_fit_err = amplitude_fit_err scatter_fraction_fit_err = perr[6:5+N]+[np.sqrt(sum(np.array(perr[6:5+N])**2))] @@ -3284,9 +3284,9 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, output_string += '-----------------\n' output_string += 'survival probability = {:.8e}'.format(survival_probability_fit) + ' +/- {:.8e}\n'.format(survival_probability_fit_err) output_string += '-----------------\n' - output_string += 'scatter_peak_ratio_b = {:.8e}'.format(scatter_peak_ratio_b_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_b_fit_err) + output_string += 'scatter_peak_ratio_p = {:.8e}'.format(scatter_peak_ratio_p_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_p_fit_err) output_string += '-----------------\n' - output_string += 'scatter_peak_ratio_c = {:.8e}'.format(scatter_peak_ratio_c_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_c_fit_err) + output_string += 'scatter_peak_ratio_q = {:.8e}'.format(scatter_peak_ratio_q_fit) + ' +/- {:.8e}\n'.format(scatter_peak_ratio_q_fit_err) output_string += '-----------------\n' for i in range(len(self.gases)): output_string += '{} scatter fraction \n= '.format(self.gases[i]) + "{:.8e}".format(scatter_fraction_fit[i])\ @@ -3305,10 +3305,10 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, 'B_field_fit_err': B_field_fit_err, 'scale_factor_fit': scale_factor_fit, 'scale_factor_fit_err': scale_factor_fit_err, - 'scatter_peak_ratio_b_fit': scatter_peak_ratio_b_fit, - 'scatter_peak_ratio_b_fit_err': scatter_peak_ratio_b_fit_err, - 'scatter_peak_ratio_c_fit': scatter_peak_ratio_c_fit, - 'scatter_peak_ratio_c_fit_err': scatter_peak_ratio_c_fit_err, + 'scatter_peak_ratio_p_fit': scatter_peak_ratio_p_fit, + 'scatter_peak_ratio_p_fit_err': scatter_peak_ratio_p_fit_err, + 'scatter_peak_ratio_q_fit': scatter_peak_ratio_q_fit, + 'scatter_peak_ratio_q_fit_err': scatter_peak_ratio_q_fit_err, 'amplitude_fit': amplitude_fit, 'amplitude_fit_err': amplitude_fit_err, 'data_hist_freq': data_hist_freq, From c6ebe4ff1e9616a1a5673a6d358ececb04f737dd Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 20 Oct 2021 15:00:34 -0400 Subject: [PATCH 141/199] modify test_analysis/Complex_line_shape_fitter.py --- test_analysis/Complex_line_shape_fitter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 25916376..597fabad 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -32,6 +32,10 @@ def test_complex_lineshape(self): complexLineShape_config = { 'bins_choice': np.linspace(0e6, 100e6, 1000), 'gases': ["H2", "He", "Ar", "Kr"], # Ar, Kr + 'fix_gas_composition': True, + 'fix_width_scale_factor': True, + 'factor': 0.4934, + 'scatter_fractions_for_gases': [0.817, 0.07, 0.08], 'max_scatters': 20, # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, 'simulated_resolution_scaled_fit_scatter_peak_ratio', 'gaussian_resolution_fit_scatter_peak_ratio' 'resolution_function': 'simulated_resolution_scaled_fit_scatter_peak_ratio', From 209372ed56d62a5166ff8744d6423a4d3c8fcac0 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Wed, 20 Oct 2021 15:17:24 -0400 Subject: [PATCH 142/199] update mermithid/processors/misc/MultiGasComplexLineShape.py --- .../processors/misc/MultiGasComplexLineShape.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 941a0468..c54226bc 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -3156,8 +3156,8 @@ def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(self, bins_ amplitude = p0[1] scale_factor = p0[2] survival_probability = p0[3] - scatter_peak_ratio_b = p0[4] - scatter_peak_ratio_c = p0[5] + scatter_peak_ratio_p = p0[4] + scatter_peak_ratio_q = p0[5] N = len(self.gases) scatter_fraction = p0[6:5+N] @@ -3197,7 +3197,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) - else: + else: eff_array = np.ones(len(bins_Hz)) # Initial guesses for curve_fit B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) @@ -3515,7 +3515,7 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi return dictionary_of_fit_results - def generate_scatter_peaks(self, emitted_peak = self.base_shape): + def generate_scatter_peaks(self): p = np.zeros(len(self.gases)) scatter_fraction = self.scatter_fractions_for_gases @@ -3527,6 +3527,7 @@ def generate_scatter_peaks(self, emitted_peak = self.base_shape): en_array = self.std_eV_array() scatter_peaks = np.zeros((self.max_scatters+1, len(en_array))) + emitted_peak = self.base_shape if emitted_peak == 'lorentzian': current_working_spectrum = self.std_lorenztian_17keV() elif emitted_peak == 'shake': @@ -3573,8 +3574,8 @@ def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_ B_field = p0[0] amplitude = p0[1] survival_probability = p0[2] - scatter_peak_ratio_b = p0[3] - scatter_peak_ratio_c = p0[4] + scatter_peak_ratio_p = p0[3] + scatter_peak_ratio_q = p0[4] x_eV = ConversionFunctions.Energy(bins_Hz, B_field) en_loss_array = self.std_eV_array() @@ -3587,7 +3588,7 @@ def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_ zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - full_spectrum = self.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(scatter_peaks, survival_probability, scatter_peak_ratio_b, scatter_peak_ratio_c) + full_spectrum = self.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(scatter_peaks, survival_probability, scatter_peak_ratio_p, scatter_peak_ratio_q) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f_intermediate = f_intermediate*eff_array f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) From 7fa0461d55e29bc61cf91652fa4d59b163401551 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Wed, 20 Oct 2021 23:11:32 -0400 Subject: [PATCH 143/199] Scatter peak ratio re-parameterization in fake data gen --- mermithid/misc/FakeTritiumDataFunctions.py | 13 +++++++------ .../TritiumSpectrum/FakeDataGenerator.py | 19 +++++++++++-------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 7d53fa44..3c4bade1 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -277,7 +277,7 @@ def convolved_bkgd_rate(K, Kmin, Kmax, lineshape, ls_params, min_energy, max_ene #Convolution of signal and lineshape using scipy.signal.convolve def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, - lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, min_energy, max_energy, + lineshape, ls_params, scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, min_energy, max_energy, complexLineShape, final_state_array, resolution_function, ins_res_width_bounds, ins_res_width_factors): """K is an array-like object """ @@ -309,9 +309,10 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape_rates = [] scale_factors = [ls_params[0]*f for f in ins_res_width_factors] for scale in scale_factors: - lineshape_rates.append(np.flipud(complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale, ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac'))) + lineshape_rates.append(np.flipud(complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale, ls_params[1], scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, emitted_peak='dirac'))) elif resolution_function == 'gaussian_resolution' or resolution_function == 'gaussian': - lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') + logger.warn("Scatter peak ratio function for lineshape with Gaussian resolution may not be up-to-date!") + lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, emitted_peak='dirac') else: logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) @@ -338,7 +339,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, #Convolution of background and lineshape using scipy.signal.convolve -def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, min_energy, max_energy, complexLineShape, resolution_function): +def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, min_energy, max_energy, complexLineShape, resolution_function): """K is an array-like object """ energy_half_range = max(max_energy, abs(min_energy)) @@ -355,9 +356,9 @@ def convolved_bkgd_rate_arrays(K, Kmin, Kmax, lineshape, ls_params, scatter_peak lineshape_rates = simplified_ls(K_lineshape, 0, ls_params[0], ls_params[1], ls_params[2], ls_params[3], ls_params[4], ls_params[5]) elif lineshape=='detailed_scattering' or lineshape=='detailed': if resolution_function == 'simulated_resolution' or resolution_function == 'simulated': - lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') + lineshape_rates = complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_p, scatter_peak_ratio_p, scatter_fraction, emitted_peak='dirac') elif resolution_function == 'gaussian_resolution' or resolution_function == 'gaussian': - lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='dirac') + lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, emitted_peak='dirac') else: logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) lineshape_rates = np.flipud(lineshape_rates) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index dc54fa32..21ea6641 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -54,8 +54,9 @@ def InternalConfigure(self, params): – gases: list of strings naming gases to be included in complex lineshape model. Options: 'H2', 'He', 'Kr', 'Ar', 'CO' - NScatters: lineshape parameter - number of scatters included in lineshape - trap_weights: distionary of two lists, labeled 'weights' and 'errors', which respectively include the fractions of counts from each trap and the uncertainties on those fractions - - scatter_peak_ratio_b: "b" in reconstrudction efficiency curve model: e^(-b*i^c), where i is the scatter order - - scatter_peak_ratio_c: "c" in the same reconstruction efficiency model + - scatter_peak_ratio_p: "p" in reconstrudction efficiency curve model: e^(-p*i^(-factor*p+q)), where i is the scatter order + - scatter_peak_ratio_q: "q" in the same reconstruction efficiency model + - scatter_peak_ratio_factor: "factor" in the same reconstruction efficiency model – scatter_proportion: list of proportion of scatters due to each gas in self.gases (in the same order), in complex lineshape - survival_prob: lineshape parameter - probability of electron staying in the trap between two inelastics scatters (it could escape due to elastics scatters or the inelastics scatters, themselves) – use_radiation_loss: if True, radiation loss will be included in the complex lineshape; should be set to True except for testing purposes @@ -127,8 +128,9 @@ def InternalConfigure(self, params): self.NScatters = reader.read_param(params, 'NScatters', 20) self.trap_weights = reader.read_param(params, 'trap_weights', {'weights':[0.076, 0.341, 0.381, 0.203], 'errors':[0.003, 0.013, 0.014, 0.02]}) #self.recon_eff_params = reader.read_param(params, 'recon_eff_params', [0.005569990343215976, 0.351, 0.546]) - self.scatter_peak_ratio_b = reader.read_param(params, 'scatter_peak_ratio_b', 0.686312493) - self.scatter_peak_ratio_c = reader.read_param(params, 'scatter_peak_ratio_c', 0.52481056) + self.scatter_peak_ratio_p = reader.read_param(params, 'scatter_peak_ratio_p', 1.) + self.scatter_peak_ratio_q = reader.read_param(params, 'scatter_peak_ratio_q', 0.6) + self.scatter_peak_ratio_factor = reader.read_param(params, 'scatter_peak_ratio_factor', 0.5) self.scatter_proportion = reader.read_param(params, 'scatter_proportion', []) self.survival_prob = reader.read_param(params, 'survival_prob', 1.) self.use_radiation_loss = reader.read_param(params, 'use_radiation_loss', True) @@ -214,8 +216,9 @@ def InternalConfigure(self, params): 'use_radiation_loss': self.use_radiation_loss, 'sample_ins_res_errors': self.sample_ins_resolution_errors, 'resolution_function': self.resolution_function, - 'scatter_peak_ratio_b': self.scatter_peak_ratio_b, - 'scatter_peak_ratio_c': self.scatter_peak_ratio_c, + 'scatter_peak_ratio_p': self.scatter_peak_ratio_p, + 'scatter_peak_ratio_q': self.scatter_peak_ratio_q, + 'factor': self.scatter_peak_ratio_factor, 'fit_recon_eff': self.fit_recon_eff, #For analytics resolution functions, only: @@ -386,7 +389,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 if array_method == True: ratesS = convolved_spectral_rate_arrays(self.Koptions, Q_mean, - mass, Kmin, lineshape, params, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, self.scatter_proportion, min_energy, max_energy, + mass, Kmin, lineshape, params, self.scatter_peak_ratio_p, self.scatter_peak_ratio_q, self.scatter_proportion, min_energy, max_energy, self.complexLineShape, self.final_state_array, self.resolution_function, self.ins_res_width_bounds, self.ins_res_width_factors) else: ratesS = [convolved_spectral_rate(K, Q_mean, mass, Kmin, @@ -402,7 +405,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 # background if array_method == True: ratesB = convolved_bkgd_rate_arrays(self.Koptions, Kmin, Kmax, - lineshape, params, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, self.scatter_proportion, min_energy, max_energy, + lineshape, params, self.scatter_peak_ratio_p, self.scatter_peak_ratio_q, self.scatter_proportion, min_energy, max_energy, self.complexLineShape, self.resolution_function) else: ratesB = [convolved_bkgd_rate(K, Kmin, Kmax, lineshape, params, From 746071d97fcbfc2f3eed1fd7e0a2a84c6be4881b Mon Sep 17 00:00:00 2001 From: cclaessens Date: Fri, 22 Oct 2021 10:37:08 -0700 Subject: [PATCH 144/199] modified simplified lineshape and added channel livetime correction --- .../processors/Fitters/BinnedDataFitter.py | 2 +- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 174 ++++++++++-------- 2 files changed, 102 insertions(+), 74 deletions(-) diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index 046b96f7..24c8d283 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -186,7 +186,7 @@ def PoissonLogLikelihood(self, params): logger.error('FYI, the parameters are: {}'.format(params)) # exclude bins where expectation is <= zero or nan - index = np.where(expectation>0) + index = np.where(expectation>0)#np.finfo(0.0).resolution) # poisson log likelihoood ll = (self.hist[index]*np.log(expectation[index]) - expectation[index]).sum() diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 82fcd8eb..ebd4359e 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -157,6 +157,7 @@ def InternalConfigure(self, config_dict): self.integrate_bins = reader.read_param(config_dict, 'integrate_bins', True) # integrate spectrum over bin widths self.fit_efficiency_tilt = reader.read_param(config_dict, 'fit_efficiency_tilt', False) # efficiency slope is free parameter self.fit_nu_mass = reader.read_param(config_dict, 'fit_neutrino_mass', False) + self.use_relative_livetime_correction = reader.read_param(config_dict, 'use_relative_livetime_correction', False) # detector response options self.NScatters = reader.read_param(config_dict, 'NScatters', 20) @@ -193,7 +194,8 @@ def InternalConfigure(self, config_dict): # check that configuration is consistent if (len(self.model_parameter_names) != len(self.model_parameter_means) or len(self.model_parameter_names) != len(self.model_parameter_widths) or len(self.model_parameter_names) != len(self.fixed_parameters)): - raise IndexError('Number of parameter names does not match other parameter configurations') + logger.error('Number of parameter names does not match other parameter configurations') + return False # ==================================== @@ -222,11 +224,6 @@ def InternalConfigure(self, config_dict): self.res_mean = reader.read_param(config_dict, 'gaussian_resolution_mean', 15.0) self.res_width = reader.read_param(config_dict, 'gaussian_resolution_width', 1.0) self.res = self.res_mean - if self.res <= 30.01/float(2*np.sqrt(2*np.log(2))): - logger.warning('Resolution small for shallow trap model. Setting to {}'.format(30.01/float(2*np.sqrt(2*np.log(2))))) - self.res = 30.01/float(2*np.sqrt(2*np.log(2))) - - if self.resolution_model != 'gaussian': self.two_gaussian_sigma_1_index = self.model_parameter_names.index('two_gaussian_sigma_1') @@ -300,14 +297,21 @@ def InternalConfigure(self, config_dict): self.endpoint_mean = reader.read_param(config_dict,'endpoint_mean', self.model_parameter_means[self.endpoint_index]) self.endpoint_width = reader.read_param(config_dict, 'endpoint_width', self.model_parameter_widths[self.endpoint_index]) - # frequency range + # frequency an energy range self.min_frequency = reader.read_param(config_dict, 'min_frequency', "required") self.max_frequency = reader.read_param(config_dict, 'max_frequency', None) - + self.max_energy = reader.read_param(config_dict, 'resolution_energy_max', 1200) # path to json with efficiency dictionary self.efficiency_file_path = reader.read_param(config_dict, 'efficiency_file_path', '') + # channel livetimes + self.channel_transition_freqs = np.array(reader.read_param(config_dict, 'channel_transition_freqs', [[0,1.38623121e9+24.5e9], + [1.38623121e9+24.5e9, 1.44560621e9+24.5e9], + [1.44560621e9+24.5e9, 50e9]])) + self.channel_livetimes = reader.read_param(config_dict, 'channel_livetimes', [7185228, 7129663, 7160533]) + self.channel_relative_livetimes = np.array(self.channel_livetimes)/ np.max(self.channel_livetimes) + @@ -488,8 +492,8 @@ def InternalConfigure(self, config_dict): self.multi_gas_lineshape = self.complex_lineshape else: - self.multi_gas_lineshape = self.more_accurate_simplified_multi_gas_lineshape - logger.info('Using default lineshape: more-accurate-simplified') + logger.error('Unknown configure lineshape') + return False # scatter peak ratio self.scatter_peak_ratio = reader.read_param(config_dict, 'scatter_peak_ratio', 'modified_exponential') @@ -524,6 +528,7 @@ def InternalConfigure(self, config_dict): # adjust energy stepsize to match bin division self.denergy = self.dbins/np.round(self.dbins/self.denergy) + self._energies = np.arange(self.Energy(self.max_frequency), self.Energy(self.max_frequency)+(self.N_energy_bins)*self.denergy, self.denergy) self._bins = np.arange(np.min(self.energies), self.Energy(self.max_frequency)+(self.N_bins)*self.dbins, self.dbins) @@ -543,7 +548,7 @@ def InternalConfigure(self, config_dict): self._bin_efficiency, self._bin_efficiency_errors = self.Efficiency(self.bin_centers) self._full_efficiency, self._full_efficiency_errors = self.Efficiency(self.energies) - + self.ReSetBins() # ================================== # configure parent BinnedDataFitter # ================================== @@ -696,9 +701,9 @@ def SamplePriors(self, sampled_parameters): sample_values = [] if 'resolution' in sampled_parameters.keys() and sampled_parameters['resolution']: self.res = self.Gaussian_sample(self.res_mean, self.res_width) - if self.res <= 30.01/float(2*np.sqrt(2*np.log(2))): - logger.warning('Sampled resolution small. Setting to {}'.format(30.01/float(2*np.sqrt(2*np.log(2))))) - self.res = 30.01/float(2*np.sqrt(2*np.log(2))) + #if self.res <= 30.01/float(2*np.sqrt(2*np.log(2))): + # logger.warning('Sampled resolution small. Setting to {}'.format(30.01/float(2*np.sqrt(2*np.log(2))))) + # self.res = 30.01/float(2*np.sqrt(2*np.log(2))) self.parameter_samples['resolution'] = self.res sample_values.append(self.res) if 'two_gaussian_sigma_1' in sampled_parameters.keys() and sampled_parameters['two_gaussian_sigma_1']: @@ -851,6 +856,7 @@ def ReSetBins(self): #self.energies = np.arange(self.Energy(self.max_frequency), self.Energy(self.min_frequency), self.denergy) #self.bins = np.arange(np.min(self.energies), np.max(self.energies), self.dbins) + #self._bin_efficiency, self._bin_efficiency_error = [], [] self.energies = np.arange(self.Energy(self.max_frequency), self.Energy(self.max_frequency)+(self.N_energy_bins)*self.denergy, self.denergy) self.bins = np.arange(np.min(self.energies), np.min(self.energies)+(self.N_bins)*self.dbins, self.dbins) self._bin_efficiency, self._bin_efficiency_error = [], [] @@ -860,6 +866,10 @@ def ReSetBins(self): if len(self._bins) > self.N_bins: self._bins = self._bins[:-1] + if self.use_relative_livetime_correction: + self.channel_energy_edges = self.Energy(self.channel_transition_freqs) + #logger.info('Channel energy edges: {}'.format(self.channel_energy_edges)) + # ========================================================================= @@ -868,7 +878,6 @@ def ReSetBins(self): def GenerateData(self, params, N): - #print('Generating data') x = self.energies[0:-1]+0.5*(self.energies[1]-self.energies[0]) pdf = np.longdouble(self.TritiumSpectrumBackground(x, *params)) pdf[pdf<0]=0. @@ -940,12 +949,44 @@ def gauss_resolution_f(self, energy_array, A, sigma, mu): return f - def approximate_shape(self, K, Q, m_nu_squared, index): + def approximate_shape(self, K, Q, m_nu_squared):#, index): shape = np.zeros(len(K)) - nu_mass_shape = ((Q - K[index])**2 -m_nu_squared)**0.5 - shape[index] = (Q - K[index])*nu_mass_shape + nu_mass_shape_squared = (Q - K)**2 -m_nu_squared + Q_minus_K = Q-K + index = np.where((nu_mass_shape_squared>0) & (Q_minus_K>0)) + nu_mass_shape = np.sqrt(nu_mass_shape_squared[index]) + + shape[index] = (Q_minus_K[index])*nu_mass_shape return shape + def chopped_approximate_spectrum(self, E, Q, m_nu_squared): + + if self.use_final_states: + N_states = len(self.final_state_array[0]) + Q_states = Q+self.final_state_array[0]-np.max(self.final_state_array[0]) + approximate_e_phase_space = self.ephasespace(E, Q) + + beta_rates_array = [self.approximate_shape(E, Q_states[i], m_nu_squared)#, index[i]) + * self.final_state_array[1][i] + * approximate_e_phase_space for i in range(N_states)] + + to_return = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) + + else: + approximate_e_phase_space = self.ephasespace(E, Q) + beta_rates_array = self.approximate_shape(E, Q, m_nu_squared) * approximate_e_phase_space + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3) * beta_rates_array + + channel_a_index = np.where((Eself.channel_energy_edges[0][1])) + channel_b_index = np.where((Eself.channel_energy_edges[1][1])) + channel_c_index = np.where((Eself.channel_energy_edges[2][1])) + + spectrum[channel_a_index] = spectrum[channel_a_index]*self.channel_relative_livetimes[0] + spectrum[channel_b_index] = spectrum[channel_b_index]*self.channel_relative_livetimes[1] + spectrum[channel_c_index] = spectrum[channel_c_index]*self.channel_relative_livetimes[2] + + return spectrum + def approximate_spectrum(self, E, Q, m_nu_squared=0): """ @@ -954,51 +995,35 @@ def approximate_spectrum(self, E, Q, m_nu_squared=0): but the ephasespace is approximate and neutrino parameter is mass squared (some factors neglected) """ - # mnu is used in heaviside function - #if m_nu_squared >=0: - mnu = np.abs(m_nu_squared)**0.5 - #else: - # mnu = 0 if self.use_final_states: - if isinstance(E, list) or isinstance(E, np.ndarray): - N_states = len(self.final_state_array[0]) - Q_states = Q+self.final_state_array[0]-np.max(self.final_state_array[0]) - approximate_e_phase_space = self.ephasespace(E, Q) - - index = [np.where(((Q_states[i]-E)**2-m_nu_squared > 0) & (Q_states[i]-E > 0)) for i in range(N_states)] - #index = [np.where(E < Q_states[i] -mnu) for i in range(N_states)] - beta_rates_array = [self.approximate_shape(E, Q_states[i], m_nu_squared, index[i]) - * self.final_state_array[1][i] - * approximate_e_phase_space for i in range(N_states)] - - to_return = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) - return to_return - - else: - logger.warning('E is not array!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') + N_states = len(self.final_state_array[0]) + Q_states = Q+self.final_state_array[0]-np.max(self.final_state_array[0]) + approximate_e_phase_space = self.ephasespace(E, Q) - return_value = 0. + #index = [np.where(((Q_states[i]-E)**2-m_nu_squared > 0) & (Q_states[i]-E > 0)) for i in range(N_states)] + #index = [np.where(E < Q_states[i] -mnu) for i in range(N_states)] + beta_rates_array = [self.approximate_shape(E, Q_states[i], m_nu_squared)#, index[i]) + * self.final_state_array[1][i] + * approximate_e_phase_space for i in range(N_states)] - for i, e_binding in enumerate(self.final_state_array[0]): - # binding energies are negative - Q_state = Q+e_binding - if Q_state-mnu > E > 0: - return_value += self.final_state_array[1][i] *(GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.ephasespace(E, Q_state)* - (Q_state - E)*np.sqrt((Q_state - E)**2 - (mnu)**2)) - - return return_value/np.sum(self.final_state_array[1]) + to_return = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) else: - beta_rates = np.zeros(len(E)) + approximate_e_phase_space = self.ephasespace(E, Q) + #index = np.where(((Q-E)**2-m_nu_squared > 0) & (Q-E > 0)) + beta_rates_array = self.approximate_shape(E, Q, m_nu_squared) * approximate_e_phase_space + + #beta_rates = np.zeros(len(E)) - index = np.where(E < Q-mnu) - K = E[index] + #index = np.where(E < Q-mnu) + #K = E[index] - nu_mass_shape = ((Q - K)**2 -m_nu_squared)**0.5 - beta_rates[index] = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.ephasespace(K, Q)*(Q - K)*nu_mass_shape + #nu_mass_shape = ((Q - K)**2 -m_nu_squared)**0.5 + #beta_rates[index] = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.ephasespace(K, Q)*(Q - K)*nu_mass_shape + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3) * beta_rates_array - return beta_rates + return spectrum def ephasespace(self, K, Q): #G = rad_corr(K, Q) #Radiative correction @@ -1011,6 +1036,8 @@ def ephasespace(self, K, Q): return pe(K)*Ee(K)*F#*G*S*I*R*LC*X def which_model(self, *pars): + if self.use_relative_livetime_correction: + return self.chopped_approximate_spectrum(*pars) if self.use_approx_model: return self.approximate_spectrum(*pars) else: @@ -1068,7 +1095,7 @@ def more_accurate_simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, - sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) + # sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) #shape0 = self.gauss_resolution_f(K, 1, sig0, Kcenter) #shape0 *= 1/np.sum(shape0) shape0 = np.zeros(len(K)) @@ -1079,6 +1106,9 @@ def more_accurate_simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, hydrogen_scattering = np.zeros(len(K)) helium_scattering = np.zeros(len(K)) + if FWHM < 30: + FWHM = 30 + #plt.figure(figsize=(10,10)) for i in range(self.NScatters): @@ -1133,9 +1163,13 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): q0, q1, q2, q3 = self.helium_lineshape_p[1], self.helium_lineshape_p[3], self.helium_lineshape_p[5], self.helium_lineshape_p[7] + sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) + if sig0 < 6: + logger.warning('Scatter resolution < 6 eV. Setting to 6 eV') + sig0 = 6 #shape = self.gauss_resolution_f(K, 1, sig0, Kcenter) - shape = np.zeros(len(K)) + #shape = np.zeros(len(K)) norm = 1. hydrogen_scattering = np.zeros(len(K)) @@ -1146,8 +1180,8 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): for i in range(self.NScatters): # hydrogen scattering - sig = p0[i]+p1[i]*FWHM - mu = -(p2[i]+p3[i]*np.log(FWHM-30)) + mu = -p0[i]+p1[i]*sig0 + sig = p2[i]+p3[i]*sig0 if self.use_fixed_scatter_peak_ratio: probi = prob_b**(i+1) @@ -1160,8 +1194,8 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): #plt.plot(K, h_scatter_i, color='blue', label='hydrogen') # helium scattering - mu_he = -(q0[i]+q1[i]*FWHM) - sig_he = q2[i]+q3[i]*FWHM + mu_he = -q0[i]+q1[i]*sig0 + sig_he = q2[i]+q3[i]*sig0 he_scatter_i = probi*self.gauss_resolution_f(K, 1, sig_he, mu_he+Kcenter) helium_scattering += he_scatter_i @@ -1170,7 +1204,7 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): #plt.plot(K, (shape + hydrogen_scattering)/np.max(shape + hydrogen_scattering), color='blue', label='hydrogen') #plt.plot(K, (shape + helium_scattering)/np.max(shape + helium_scattering), color='red', label='helium') # full lineshape - lineshape = (shape + self.hydrogen_proportion*hydrogen_scattering + (1-self.hydrogen_proportion)*helium_scattering) + lineshape = self.hydrogen_proportion*hydrogen_scattering + (1-self.hydrogen_proportion)*helium_scattering #plt.plot(K, lineshape/np.max(lineshape), color='black', label='full') #plt.xlim(-200, 200) @@ -1234,7 +1268,7 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p sig2 = self.two_gaussian_sigma_2 - max_energy = 1000 + max_energy = self.max_energy dE = self.energies[1]-self.energies[0]#E[1]-E[0] n_dE = round(max_energy/dE) #e_add = np.arange(np.min(self.energies)-round(max_energy/dE)*dE, np.min(self.energies), dE) @@ -1281,10 +1315,6 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p # simplified lineshape FWHM = 2.*np.sqrt(2.*np.log(2.))*res *self.width_scaling - if FWHM < 30.01: - #logger.warning('FWHM smaller 30. Setting to 30 instead.') - FWHM = 30.01 - # get lineshape if not self.use_helium_scattering: @@ -1336,9 +1366,7 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p FWHM = 2.*np.sqrt(2.*np.log(2.))*res*self.width_scaling - if FWHM < 30.01: - logger.warning('FWHM smaller 30. Setting to 30 instead.') - FWHM = 30.01 + #logger.info('Plotting lineshape for FWHM {} and hydrogen proportion {}.'.format(FWHM, self.hydrogen_proportion)) #simple_ls, simple_norm = self.simplified_ls(e_lineshape, 0, FWHM, prob_b, prob_c) #simple_ls = (self.gauss_resolution_f(e_lineshape, 1, res, 0)+simple_ls)/simple_norm @@ -1391,15 +1419,15 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p K_convolved = np.interp(E, self._energies, K_convolved)*(len(E)*1./len(self._energies)) - # multiply efficiency - efficiency = np.ones(len(E)) - efficiency_errors = [np.zeros(len(E)), np.zeros(len(E))] - if self.is_distorted == True: #if self.fit_efficiency_tilt: # self.tilt = tilt efficiency, efficiency_errors = self.Efficiency(E) + else: + # multiply efficiency + efficiency = np.ones(len(E)) + efficiency_errors = [np.zeros(len(E)), np.zeros(len(E))] K_eff=K_convolved*efficiency # finally From 1109f4ff2754eaa8bd20e7f8983348c322f75d02 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Fri, 22 Oct 2021 16:27:46 -0400 Subject: [PATCH 145/199] add configurable detection eff --- mermithid/processors/misc/MultiGasComplexLineShape.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index c54226bc..062ea131 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -3609,9 +3609,12 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) - quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] - eff_array = quad_trap_count_rate_interp(bins_Hz) + if self.use_quad_trap_eff_interp == True: + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) + quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] + eff_array = quad_trap_count_rate_interp(bins_Hz) + else: + eff_array = np.ones(len(bins_Hz)) # Initial guesses for curve_fit B_field_guess = ComplexLineShapeUtilities.central_frequency_to_B_field(bins_Hz[np.argmax(data_hist_freq)]) amplitude_guess = np.sum(data_hist_freq) From 55663e599f8dbbb23d3bbc3716ba6a8b5117dede Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Fri, 22 Oct 2021 16:44:51 -0400 Subject: [PATCH 146/199] update test_analysis/Complex_line_shape_fitter.py --- test_analysis/Complex_line_shape_fitter.py | 37 +++++++++++----------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index 597fabad..db0c030a 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -31,27 +31,12 @@ def test_complex_lineshape(self): complexLineShape_config = { 'bins_choice': np.linspace(0e6, 100e6, 1000), - 'gases': ["H2", "He", "Ar", "Kr"], # Ar, Kr + 'gases': ["H2", "He"], # "Ar", "Kr" # "Kr" for fss 'fix_gas_composition': True, 'fix_width_scale_factor': True, 'factor': 0.4934, - 'scatter_fractions_for_gases': [0.817, 0.07, 0.08], + 'scatter_fractions_for_gases': [0.894], 'max_scatters': 20, - # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, 'simulated_resolution_scaled_fit_scatter_peak_ratio', 'gaussian_resolution_fit_scatter_peak_ratio' - 'resolution_function': 'simulated_resolution_scaled_fit_scatter_peak_ratio', - #choose the parameters you want to fix from ['B field','amplitude','width scale factor', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] plus the gas scatter fractions as ['H2 scatter fraction'], - 'fixed_parameter_names': ['survival probability', 'width scale factor', 'H2 scatter fraction', 'He scatter fraction', 'Ar scatter fraction'], - 'fixed_parameter_values': [1.0, 1.0, 0.817, 0.07, 0.08], - # This is an important parameter which determines how finely resolved - # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown - 'num_points_in_std_array': 4000, - 'RF_ROI_MIN': 25859375000.0, - # shake_spectrum_parameters.json and oscillator strength data can be found at https://github.com/project8/scripts/tree/master/yuhao/line_shape_fitting/data - 'shake_spectrum_parameters_json_path': '../mermithid/misc/shake_spectrum_parameters.json', - 'path_to_osc_strengths_files': '/host/', - 'path_to_scatter_spectra_file': '/host/', - 'path_to_ins_resolution_data_txt': '/host/October_FTC_resolution/all_res_cf14.400.txt', - 'fixed_scatter_proportion': True, # When fixed_scatter_proportion is True, set the scatter proportion for the gases below 'gas_scatter_proportion': [0.8, 0.2],#0.827, 0.076, 0.068, 0.028 # 0.75, 0.25 @@ -59,9 +44,13 @@ def test_complex_lineshape(self): 'free_gases': ["H2", "He"], 'fixed_gases': ["Ar", "Kr"], 'scatter_proportion_for_fixed_gases': [0.018, 0.039], + 'use_radiation_loss': True, + 'sample_ins_res_errors': False, 'fixed_survival_probability': False, # When option fixed_survival_probability is True, assign the survival probability below 'survival_prob': 15/16., # assuming total cross section for elastic scattering is 1/10 of inelastic scattering + # configure the resolution functions: simulated_resolution, gaussian_resolution, gaussian_lorentzian_composite_resolution, elevated_gaussian, composite_gaussian, composite_gaussian_pedestal_factor, composite_gaussian_scaled, simulated_resolution_scaled, 'simulated_resolution_scaled_fit_scatter_peak_ratio', 'gaussian_resolution_fit_scatter_peak_ratio' + 'resolution_function': 'simulated_resolution_scaled_fit_scatter_peak_ratio', # specific choice of parameters in the gaussian lorentzian composite resolution function 'recon_eff_param_a': 0.005569990343215976, 'recon_eff_param_b': 0.351, @@ -73,7 +62,19 @@ def test_complex_lineshape(self): 'A_array': [0.076, 0.341, 0.381, 0.203], #parameter for simulated resolution scaled resolution 'fit_recon_eff': False, - #parameters for simulated resolution scaled with scatter peak ratio fitted + #parameters for simulated resolution scaled with scatter peak ratio fitted + #choose the parameters you want to fix from ['B field','amplitude', 'width scale factor', 'survival probability','scatter peak ratio param b', 'scatter peak ratio param c'] plus the gas scatter fractions as ['H2 scatter fraction'], + 'fixed_parameter_names': ['survival probability'], #, 'width scale factor', 'H2 scatter fraction', 'He scatter fraction', 'Ar scatter fraction' + 'fixed_parameter_values': [1.0], #[1.0, 1.0, 0.886, 0.02, 0.06] + # This is an important parameter which determines how finely resolved + # the scatter calculations are. 10000 seems to produce a stable fit, with minimal slowdown + 'num_points_in_std_array': 4000, + 'RF_ROI_MIN': 25859375000.0, #24.5e9 + 1.40812680e+09 - 50e6, #25850000000.0 + # shake_spectrum_parameters.json and oscillator strength data can be found at https://github.com/project8/scripts/tree/master/yuhao/line_shape_fitting/data + 'shake_spectrum_parameters_json_path': '../mermithid/misc/shake_spectrum_parameters.json', + 'path_to_osc_strengths_files': '/host/', + 'path_to_scatter_spectra_file': '/host/', + 'path_to_ins_resolution_data_txt': '/host/March_FTC_resolution/all_res_cf15.300.txt' } From 7915c2225d07f309e9caf49d395d348df20c5181 Mon Sep 17 00:00:00 2001 From: Yu-Hao Sun Date: Fri, 22 Oct 2021 20:55:01 -0400 Subject: [PATCH 147/199] change factor to 0.4626 --- test_analysis/Complex_line_shape_fitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_analysis/Complex_line_shape_fitter.py b/test_analysis/Complex_line_shape_fitter.py index db0c030a..43309b7d 100644 --- a/test_analysis/Complex_line_shape_fitter.py +++ b/test_analysis/Complex_line_shape_fitter.py @@ -34,7 +34,7 @@ def test_complex_lineshape(self): 'gases': ["H2", "He"], # "Ar", "Kr" # "Kr" for fss 'fix_gas_composition': True, 'fix_width_scale_factor': True, - 'factor': 0.4934, + 'factor': 0.4626, 'scatter_fractions_for_gases': [0.894], 'max_scatters': 20, 'fixed_scatter_proportion': True, From f23425391303130a8c7fb8728d2c9cf8a868b721 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Fri, 22 Oct 2021 18:08:24 -0700 Subject: [PATCH 148/199] added fixed parameter configuraion as dictionary --- mermithid/processors/Fitters/BinnedDataFitter.py | 7 ++++++- mermithid/processors/Fitters/MCUncertaintyPropagation.py | 9 ++++++--- .../processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 3 +++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index 24c8d283..f4df0d8f 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -57,6 +57,7 @@ def InternalConfigure(self, params): self.initial_values = reader.read_param(params, 'initial_values', [1]*len(self.parameter_names)) self.limits = reader.read_param(params, 'limits', [[None, None]]*len(self.parameter_names)) self.fixes = reader.read_param(params, 'fixed', [False]*len(self.parameter_names)) + self.fixes_dict = reader.read_param(params, 'fixed_parameter_dict', {}) self.bins = reader.read_param(params, 'bins', np.linspace(-2, 2, 100)) self.binned_data = reader.read_param(params, 'binned_data', False) self.print_level = reader.read_param(params, 'print_level', 1) @@ -131,7 +132,11 @@ def fit(self): m_binned.print_level = self.print_level for i, name in enumerate(self.parameter_names): - m_binned.fixed[name] = self.fixes[i] + if name in self.fixes_dict.keys(): + m_binned.fixed[name]= self.fixes_dict[name] + logger.info('Fixing {}'.format(name)) + else: + m_binned.fixed[name] = self.fixes[i] m_binned.limits[name] = self.limits[i] # minimze diff --git a/mermithid/processors/Fitters/MCUncertaintyPropagation.py b/mermithid/processors/Fitters/MCUncertaintyPropagation.py index 3ab6aa94..eb7b01ad 100644 --- a/mermithid/processors/Fitters/MCUncertaintyPropagation.py +++ b/mermithid/processors/Fitters/MCUncertaintyPropagation.py @@ -44,6 +44,8 @@ def InternalConfigure(self, params): self.sample_parameters = reader.read_param(params, 'sample_parameters', []) self.stat_sys_combined = reader.read_param(params, 'stat_sys_combined', [True, True, True]) self.N = reader.read_param(params, 'N_samples', 50) + self.fitted_params = reader.read_param(params, "initial_best_fit", []) + self.fitted_params_errors = reader.read_param(params, "initial_best_fit_errors", []) return True @@ -54,7 +56,8 @@ def InternalRun(self): for k in self.fit_options.keys(): self.fit_config_dict[k] = self.fit_options[k] - self.InitialFit() + if len(self.fitted_params) == 0 or len(self.fitted_params_errors) == 0: + self.InitialFit() self.ParameterSampling() @@ -88,8 +91,8 @@ def InitialFit(self): logger.info('Best fit: {}'.format(self.fitted_params)) - x, pdf, bins, fitted_model, asimov_data = self.model(self.fit_config_dict, - params=self.fitted_params) + #x, pdf, bins, fitted_model, asimov_data = self.model(self.fit_config_dict, + # params=self.fitted_params) return True def ParameterSampling(self): diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index ebd4359e..2d4f5bd7 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -180,6 +180,7 @@ def InternalConfigure(self, config_dict): # width for constraints or sample distributions self.model_parameter_widths = reader.read_param(config_dict, 'model_parameter_widths', [100, 0.1, 0.1, 500, 0.1, 0, 3, 1, 1]) self.fixed_parameters = reader.read_param(config_dict, 'fixed_parameters', [False, False, False, False, True, True, True, True, True]) + self.fixed_parameter_dict = reader.read_param(config_dict, 'fixed_parameter_dict', {}) self.limits = reader.read_param(config_dict, 'model_parameter_limits', [[18e3, 20e3], [0, None], @@ -586,6 +587,8 @@ def ConfigureFit(self): #self.fix_scatter_peak_ratio_b, self.fix_scatter_peak_ratio_c, #self.fix_res, self.fix_two_gaussian_sigma_1, self.fix_two_gaussian_sigma_2] + self.fixed_dict = self.fixed_parameter_dict + # self.limits = [energy_limits, # [0, None], # neutrino_limits, From f744126759126d854cabe8a067467267dbf86a68 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Fri, 22 Oct 2021 22:51:58 -0700 Subject: [PATCH 149/199] some fixes --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 2d4f5bd7..0eab69fc 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -587,7 +587,7 @@ def ConfigureFit(self): #self.fix_scatter_peak_ratio_b, self.fix_scatter_peak_ratio_c, #self.fix_res, self.fix_two_gaussian_sigma_1, self.fix_two_gaussian_sigma_2] - self.fixed_dict = self.fixed_parameter_dict + self.fixes_dict = self.fixed_parameter_dict # self.limits = [energy_limits, # [0, None], @@ -973,7 +973,7 @@ def chopped_approximate_spectrum(self, E, Q, m_nu_squared): * self.final_state_array[1][i] * approximate_e_phase_space for i in range(N_states)] - to_return = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) else: approximate_e_phase_space = self.ephasespace(E, Q) @@ -1010,7 +1010,7 @@ def approximate_spectrum(self, E, Q, m_nu_squared=0): * self.final_state_array[1][i] * approximate_e_phase_space for i in range(N_states)] - to_return = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) else: approximate_e_phase_space = self.ephasespace(E, Q) @@ -1257,18 +1257,21 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p if self.is_smeared or self.is_scattered: # resolution params - res = args[self.res_index] if self.fixed_parameters[self.res_index]: res = self.res + else: + res = args[self.res_index] if self.resolution_model != 'gaussian': - sig1 = args[self.two_gaussian_sigma_1_index] - sig2 = args[self.two_gaussian_sigma_2_index] if self.fixed_parameters[self.two_gaussian_sigma_1_index]: sig1 = self.two_gaussian_sigma_1 + else: + sig1 = args[self.two_gaussian_sigma_1_index] if self.fixed_parameters[self.two_gaussian_sigma_1_index]: sig2 = self.two_gaussian_sigma_2 + else: + sig2 = args[self.two_gaussian_sigma_2_index] max_energy = self.max_energy From c73a2ded224d102b8689e0cede4dba820b2e8c5a Mon Sep 17 00:00:00 2001 From: cclaessens Date: Tue, 16 Nov 2021 11:12:24 -0800 Subject: [PATCH 150/199] fixes for nuisance parameter analysis --- .../processors/Fitters/BinnedDataFitter.py | 4 +- .../Fitters/MCUncertaintyPropagation.py | 3 +- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 61 +++---------------- 3 files changed, 15 insertions(+), 53 deletions(-) diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index f4df0d8f..fe8e322d 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -120,6 +120,8 @@ def fit(self): logger.info('Limits: {}'.format(self.limits)) logger.info('Fixed in fit: {}'.format(self.fixes)) logger.info('Constrained parameters: {}'.format([self.parameter_names[i] for i in self.constrained_parameters])) + logger.info('Constraint means: {}'.format(self.constrained_means)) + logger.info('Constraint widths: {}'.format(self.constrained_widths)) m_binned = Minuit(self.negPoissonLogLikelihood, self.initial_values, @@ -134,7 +136,7 @@ def fit(self): for i, name in enumerate(self.parameter_names): if name in self.fixes_dict.keys(): m_binned.fixed[name]= self.fixes_dict[name] - logger.info('Fixing {}'.format(name)) + #logger.info('Fixing {}'.format(name)) else: m_binned.fixed[name] = self.fixes[i] m_binned.limits[name] = self.limits[i] diff --git a/mermithid/processors/Fitters/MCUncertaintyPropagation.py b/mermithid/processors/Fitters/MCUncertaintyPropagation.py index eb7b01ad..51d0adcd 100644 --- a/mermithid/processors/Fitters/MCUncertaintyPropagation.py +++ b/mermithid/processors/Fitters/MCUncertaintyPropagation.py @@ -46,6 +46,7 @@ def InternalConfigure(self, params): self.N = reader.read_param(params, 'N_samples', 50) self.fitted_params = reader.read_param(params, "initial_best_fit", []) self.fitted_params_errors = reader.read_param(params, "initial_best_fit_errors", []) + self.Counts = reader.read_param(params, "Counts", 0) return True @@ -56,7 +57,7 @@ def InternalRun(self): for k in self.fit_options.keys(): self.fit_config_dict[k] = self.fit_options[k] - if len(self.fitted_params) == 0 or len(self.fitted_params_errors) == 0: + if len(self.fitted_params) == 0 or len(self.fitted_params_errors) == 0 or self.Counts == 0: self.InitialFit() self.ParameterSampling() diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 0eab69fc..764b0c30 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -246,6 +246,7 @@ def InternalConfigure(self, config_dict): self.scatter_peak_ratio_b_index = self.model_parameter_names.index('scatter_peak_ratio_b') self.scatter_peak_ratio_c_index = self.model_parameter_names.index('scatter_peak_ratio_c') + self.spr_factor = reader.read_param(config_dict, 'SPR_factor', 0) self.scatter_peak_ratio_b_mean = reader.read_param(config_dict, 'scatter_peak_ratio_b_mean', 0.7) self.scatter_peak_ratio_b_width = reader.read_param(config_dict, 'scatter_peak_ratio_b_width', 0.1) self.scatter_peak_ratio_c_mean = reader.read_param(config_dict, 'scatter_peak_ratio_c_mean', 0.7) @@ -347,56 +348,10 @@ def InternalConfigure(self, config_dict): if self.nuisance_parameters[p]: self.constrained_parameters.append(i) self.constrained_means.append(self.model_parameter_means[i]) - self.constrained_widths.append(self.model_parameter_means[i]) + self.constrained_widths.append(self.model_parameter_widths[i]) if i in self.constrained_parameters: self.fixed_parameters[i] = False - # if 'res' in self.nuisance_parameters.keys(): - # self.fix_res = not self.nuisance_parameters['res'] - # self.constrained_parameters.append(6) - # self.constrained_means.append(self.res_mean) - # self.constrained_widths.append(self.res_width) - # else: - # self.fix_res = True - - # if 'scatter_peak_ratio_b' in self.nuisance_parameters.keys(): - # self.fix_scatter_peak_ratio_b = not self.nuisance_parameters['scatter_peak_ratio_b'] - # self.constrained_parameters.append(4) - # self.constrained_means.append(self.scatter_peak_ratio_b_mean) - # self.constrained_widths.append(self.scatter_peak_ratio_b_width) - # else: - # self.fix_scatter_peak_ratio_b = True - - # if 'scatter_peak_ratio_c' in self.nuisance_parameters.keys(): - # self.fix_scatter_peak_ratio_c = not self.nuisance_parameters['scatter_peak_ratio_c'] - # self.constrained_parameters.append(5) - # self.constrained_means.append(self.scatter_peak_ratio_c_mean) - # self.constrained_widths.append(self.scatter_peak_ratio_c_width) - # else: - # self.fix_scatter_peak_ratio_c = True - - # if 'two_gaussian_sigma_1' in self.nuisance_parameters.keys(): - # self.fix_two_gaussian_sigma_1 = not self.nuisance_parameters['two_gaussian_sigma_1'] - # self.constrained_parameters.append(7) - # self.constrained_means.append(self.two_gaussian_sigma_1_mean) - # self.constrained_widths.append(self.two_gaussian_sigma_1_width) - # else: - # self.fix_two_gaussian_sigma_1 = True - - # if 'two_gaussian_sigma_2' in self.nuisance_parameters.keys(): - # self.fix_two_gaussian_sigma_2 = not self.nuisance_parameters['two_gaussian_sigma_2'] - # self.constrained_parameters.append(8) - # self.constrained_means.append(self.two_gaussian_sigma_2_mean) - # self.constrained_widths.append(self.two_gaussian_sigma_2_width) - # else: - # self.fix_two_gaussian_sigma_2 = True - - # """if 'B' in self.nuisance_parameters.keys(): - # self.fix_B = not self.nuisance_parameters['B'] - # self.constrained_parameters.append(9) - # self.constrained_means.append(self.B_mean) - # self.constrained_widths.append(self.B_width)""" - # MC uncertainty propagation does not need the fit uncertainties returned by iminuit. uncertainties are instead obtained from the distribution of fit results. # But if the uncertainty is unstead propagated by adding constraiend nuisance parameters then the fit uncertainties are needed. @@ -1050,7 +1005,8 @@ def mode_exp_scatter_peak_ratio(self, prob_b, prob_c, j): ''' ratio of successive peaks taking reconstruction efficiency into account ''' - return np.exp(-prob_b*j**prob_c) + c = -self.spr_factor*prob_b + prob_c + return np.exp(-prob_b*j**c) def simplified_ls(self, K, Kcenter, FWHM, prob_b, prob_c=1): @@ -1257,19 +1213,22 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p if self.is_smeared or self.is_scattered: # resolution params - if self.fixed_parameters[self.res_index]: + if 'resolution' not in self.model_parameter_names or 'resolution' in self.parameter_samples.keys():#self.fixed_parameters[self.res_index]: res = self.res + #logger.info('Using self.res') else: res = args[self.res_index] if self.resolution_model != 'gaussian': - if self.fixed_parameters[self.two_gaussian_sigma_1_index]: + if 'two_gaussian_sigma_1' not in self.model_parameter_names or 'two_gaussian_sigma_1' in self.parameter_samples.keys(): sig1 = self.two_gaussian_sigma_1 + #logger.info('Using self.two_gaussian_sigma_1') else: sig1 = args[self.two_gaussian_sigma_1_index] - if self.fixed_parameters[self.two_gaussian_sigma_1_index]: + if 'two_gaussian_sigma_2' not in self.model_parameter_names or 'two_gaussian_sigma_2' in self.parameter_samples.keys(): sig2 = self.two_gaussian_sigma_2 + #logger.info('Using self.two_gaussian_sigma_2') else: sig2 = args[self.two_gaussian_sigma_2_index] From b533721389127c05c2005af14d610887d0658e51 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Sun, 12 Dec 2021 14:30:10 -0800 Subject: [PATCH 151/199] fixed neutrino mass bias --- .../processors/Fitters/BinnedDataFitter.py | 1 + .../Fitters/MCUncertaintyPropagation.py | 20 +++-- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 87 ++++++++++++------- 3 files changed, 68 insertions(+), 40 deletions(-) diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index fe8e322d..53866d99 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -174,6 +174,7 @@ def fit(self): if self.print_level == 1: logger.info('Fit results: {}'.format(result_array)) logger.info('Errors: {}'.format(error_array)) + #logger.info('Correlation matrix: {}'.format(self.m_binned.covariance.correlation())) return result_array, error_array diff --git a/mermithid/processors/Fitters/MCUncertaintyPropagation.py b/mermithid/processors/Fitters/MCUncertaintyPropagation.py index 51d0adcd..e936476e 100644 --- a/mermithid/processors/Fitters/MCUncertaintyPropagation.py +++ b/mermithid/processors/Fitters/MCUncertaintyPropagation.py @@ -78,6 +78,7 @@ def InitialFit(self): fit_successful = False counter = 0 + self.fit_config_dict['print_level'] = 1 #while (not fit_successful) and counter < 15: # counter += 1 # try: @@ -85,6 +86,7 @@ def InitialFit(self): self.fitted_params, self.fitted_params_errors, self.Counts = self.fit(self.data, self.fit_config_dict) fit_successful = True + self.fit_config_dict['print_level'] = 0 # except Exception as e: # print(e) # logger.error('Repeating fit') @@ -128,18 +130,18 @@ def ParameterSampling(self): for i in range(start_j, self.N): fit_successful = False counter = 0 - while (not fit_successful) and counter < 5: - counter += 1 - try: - all_fit_returns.append(self.gen_and_fit(self.fitted_params, self.Counts, + #while (not fit_successful) and counter < 5: + # counter += 1 + # try: + all_fit_returns.append(self.gen_and_fit(self.fitted_params, self.Counts, self.fit_config_dict, parameter_sampling[k_i], i, fixed_data)) - fit_successful = True - except Exception as e: - print(e) - logger.error('Repeating fit') - continue + # fit_successful = True + # except Exception as e: + # print(e) + # logger.error('Repeating fit') + # continue diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 764b0c30..991a4231 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -80,6 +80,8 @@ def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, results, errors = T.SampleConvertAndFit(sampled_priors, params) parameter_samples = T.parameter_samples + logger.info('Fit results: {}'.format(results)) + if fit_config_dict['minos_intervals']: return results, T.minos_errors, parameter_samples @@ -220,15 +222,18 @@ def InternalConfigure(self, config_dict): self.width_scaling = self.scale_mean if self.is_smeared: - self.res_index = self.model_parameter_names.index('resolution') + if 'resolution' in self.model_parameter_names: + self.res_index = self.model_parameter_names.index('resolution') self.res_mean = reader.read_param(config_dict, 'gaussian_resolution_mean', 15.0) self.res_width = reader.read_param(config_dict, 'gaussian_resolution_width', 1.0) self.res = self.res_mean if self.resolution_model != 'gaussian': - self.two_gaussian_sigma_1_index = self.model_parameter_names.index('two_gaussian_sigma_1') - self.two_gaussian_sigma_2_index = self.model_parameter_names.index('two_gaussian_sigma_2') + if 'two_gaussian_sigma_1' in self.model_parameter_names: + self.two_gaussian_sigma_1_index = self.model_parameter_names.index('two_gaussian_sigma_1') + if 'two_gaussian_sigma_2' in self.model_parameter_names: + self.two_gaussian_sigma_2_index = self.model_parameter_names.index('two_gaussian_sigma_2') self.two_gaussian_mu_1 = reader.read_param(config_dict, 'two_gaussian_mu1', 0) self.two_gaussian_mu_2 = reader.read_param(config_dict, 'two_gaussian_mu2', 0) @@ -243,8 +248,10 @@ def InternalConfigure(self, config_dict): # scatter peak ratio if self.is_scattered: - self.scatter_peak_ratio_b_index = self.model_parameter_names.index('scatter_peak_ratio_b') - self.scatter_peak_ratio_c_index = self.model_parameter_names.index('scatter_peak_ratio_c') + if 'scatter_peak_ratio_b' in self.model_parameter_names: + self.scatter_peak_ratio_b_index = self.model_parameter_names.index('scatter_peak_ratio_b') + if 'scatter_peak_ratio_c' in self.model_parameter_names: + self.scatter_peak_ratio_c_index = self.model_parameter_names.index('scatter_peak_ratio_c') self.spr_factor = reader.read_param(config_dict, 'SPR_factor', 0) self.scatter_peak_ratio_b_mean = reader.read_param(config_dict, 'scatter_peak_ratio_b_mean', 0.7) @@ -358,7 +365,7 @@ def InternalConfigure(self, config_dict): # imnuit can calculated hesse and minos intervals. the former are symmetric. we want the asymetric intrevals-> minos # minos_cls is the list of uncertainty level that should be obtained: e.g. [0.68, 0.9] self.minos_intervals = reader.read_param(config_dict, 'minos_intervals', False) - self.minos_cls = reader.read_param(config_dict, 'minos_confidence_levels', [0.683, 0.9]) + self.minos_cls = reader.read_param(config_dict, 'minos_confidence_levels', [0.683]) @@ -906,16 +913,35 @@ def gauss_resolution_f(self, energy_array, A, sigma, mu): f = A*(1/(sigma*np.sqrt(2*np.pi)))*np.exp(-(((energy_array-mu)/sigma)**2.)/2.) return f + def beta_rates(self, K, Q, mnu_squared, index): + beta_rates = np.zeros(len(K)) + nu_mass_shape = ((Q - K[index])**2 -mnu_squared)**0.5 + beta_rates[index] = nu_mass_shape*(Q - K[index]) + return beta_rates def approximate_shape(self, K, Q, m_nu_squared):#, index): - shape = np.zeros(len(K)) - nu_mass_shape_squared = (Q - K)**2 -m_nu_squared + spectrum = np.zeros(len(K)) + Q_minus_K = Q-K - index = np.where((nu_mass_shape_squared>0) & (Q_minus_K>0)) - nu_mass_shape = np.sqrt(nu_mass_shape_squared[index]) - shape[index] = (Q_minus_K[index])*nu_mass_shape - return shape + if m_nu_squared >= 0: + nu_mass_shape_squared = (Q - K)**2 -m_nu_squared + index = np.where((nu_mass_shape_squared>0) & (Q_minus_K>0)) + nu_mass_shape = np.sqrt(nu_mass_shape_squared[index]) + spectrum[index] = (Q_minus_K[index])*nu_mass_shape + else: + # mainz shape + k_squared = -m_nu_squared + mu = 0.66*np.sqrt(k_squared) + index = np.where(Q_minus_K+mu>0) + spectrum[index] = (Q_minus_K[index]+mu*np.exp(-1-Q_minus_K[index]/mu))*np.sqrt(Q_minus_K[index]**2+k_squared) + #else: + # # lanl + # k_squared = -m_nu_squared + # index = np.where(Q_minus_K > 0) + # spectrum[index] = Q_minus_K[index]**2+k_squared/2 + + return spectrum def chopped_approximate_spectrum(self, E, Q, m_nu_squared): @@ -968,19 +994,20 @@ def approximate_spectrum(self, E, Q, m_nu_squared=0): spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) else: - approximate_e_phase_space = self.ephasespace(E, Q) - #index = np.where(((Q-E)**2-m_nu_squared > 0) & (Q-E > 0)) - beta_rates_array = self.approximate_shape(E, Q, m_nu_squared) * approximate_e_phase_space - #beta_rates = np.zeros(len(E)) - #index = np.where(E < Q-mnu) - #K = E[index] + #beta_rates_array = self.approximate_shape(E, Q, m_nu_squared) * approximate_e_phase_space - #nu_mass_shape = ((Q - K)**2 -m_nu_squared)**0.5 - #beta_rates[index] = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.ephasespace(K, Q)*(Q - K)*nu_mass_shape - spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3) * beta_rates_array + + #nu_mass_shape = ((Q - E)**2 -m_nu) + #Q_minus_K = Q-E + + #spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3) * beta_rates_array + #index = np.where((nu_mass_shape>0) & (Q_minus_K>0)) + #approximate_e_phase_space = np.zeros(len(E)) + approximate_e_phase_space = self.ephasespace(E, Q) + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.approximate_shape(E, Q, m_nu_squared)*approximate_e_phase_space return spectrum def ephasespace(self, K, Q): @@ -1264,19 +1291,17 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p if self.is_scattered: # scatter params - prob_b = args[self.scatter_peak_ratio_b_index] - prob_c = args[self.scatter_peak_ratio_c_index] - - if self.fixed_parameters[self.scatter_peak_ratio_b_index]: + if 'scatter_peak_ratio_b' not in self.model_parameter_names: prob_b = self.scatter_peak_ratio_b - if self.fixed_parameters[self.scatter_peak_ratio_c_index]: prob_c = self.scatter_peak_ratio_c + else: + prob_b = args[self.scatter_peak_ratio_b_index] + prob_c = args[self.scatter_peak_ratio_c_index] - - if prob_b == None: - prob_b = self.scatter_peak_ratio_b_mean - prob_c = self.scatter_peak_ratio_c_mean - + if self.fixed_parameters[self.scatter_peak_ratio_b_index]: + prob_b = self.scatter_peak_ratio_b + if self.fixed_parameters[self.scatter_peak_ratio_c_index]: + prob_c = self.scatter_peak_ratio_c # simplified lineshape FWHM = 2.*np.sqrt(2.*np.log(2.))*res *self.width_scaling From 8b52cec03279b1452d9bf5c319b13723afb44631 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 13 Jan 2022 18:58:39 -0800 Subject: [PATCH 152/199] changed b and c to p and q --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 139 +++++++++--------- 1 file changed, 71 insertions(+), 68 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 991a4231..8a106a83 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -175,7 +175,7 @@ def InternalConfigure(self, config_dict): # configure model parameter names self.model_parameter_names = reader.read_param(config_dict, 'model_parameter_names', ['endpoint', 'background', 'm_beta_squared', 'Amplitude', - 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', + 'scatter_peak_ratio_p', 'scatter_peak_ratio_q', 'resolution', 'two_gaussian_sigma_1', 'two_gaussian_sigma_2'] ) # initial values and mean of constaints (if constraint) or mean of distribution (if sampled) self.model_parameter_means = reader.read_param(config_dict, 'model_parameter_means', [18.6e3, 0, 0, 5000, 0.8, 1, 15, 10, 10]) @@ -248,22 +248,22 @@ def InternalConfigure(self, config_dict): # scatter peak ratio if self.is_scattered: - if 'scatter_peak_ratio_b' in self.model_parameter_names: - self.scatter_peak_ratio_b_index = self.model_parameter_names.index('scatter_peak_ratio_b') - if 'scatter_peak_ratio_c' in self.model_parameter_names: - self.scatter_peak_ratio_c_index = self.model_parameter_names.index('scatter_peak_ratio_c') + if 'scatter_peak_ratio_p' in self.model_parameter_names: + self.scatter_peak_ratio_p_index = self.model_parameter_names.index('scatter_peak_ratio_p') + if 'scatter_peak_ratio_q' in self.model_parameter_names: + self.scatter_peak_ratio_q_index = self.model_parameter_names.index('scatter_peak_ratio_q') self.spr_factor = reader.read_param(config_dict, 'SPR_factor', 0) - self.scatter_peak_ratio_b_mean = reader.read_param(config_dict, 'scatter_peak_ratio_b_mean', 0.7) - self.scatter_peak_ratio_b_width = reader.read_param(config_dict, 'scatter_peak_ratio_b_width', 0.1) - self.scatter_peak_ratio_c_mean = reader.read_param(config_dict, 'scatter_peak_ratio_c_mean', 0.7) - self.scatter_peak_ratio_c_width = reader.read_param(config_dict, 'scatter_peak_ratio_c_width', 0.1) + self.scatter_peak_ratio_p_mean = reader.read_param(config_dict, 'scatter_peak_ratio_p_mean', 0.7) + self.scatter_peak_ratio_p_width = reader.read_param(config_dict, 'scatter_peak_ratio_p_width', 0.1) + self.scatter_peak_ratio_q_mean = reader.read_param(config_dict, 'scatter_peak_ratio_q_mean', 0.7) + self.scatter_peak_ratio_q_width = reader.read_param(config_dict, 'scatter_peak_ratio_q_width', 0.1) self.scatter_peak_ratio_mean = reader.read_param(config_dict, 'scatter_peak_ratio_mean', 0.5) self.scatter_peak_ratio_width = reader.read_param(config_dict, 'scatter_peak_ratio_width', 0.1) - self.scatter_peak_ratio_b = self.scatter_peak_ratio_b_mean - self.scatter_peak_ratio_c = self.scatter_peak_ratio_c_mean + self.scatter_peak_ratio_p = self.scatter_peak_ratio_p_mean + self.scatter_peak_ratio_q = self.scatter_peak_ratio_q_mean #Adding correlated parameters (will later incorporate into morpho processor) self.b_c_corr = reader.read_param(config_dict, 'b_c_corr', 0) @@ -271,7 +271,7 @@ def InternalConfigure(self, config_dict): self.c_scale_corr = reader.read_param(config_dict, 'c_scale_corr', 0) - stds = [self.scatter_peak_ratio_b_width, self.scatter_peak_ratio_c_width, self.scale_width] + stds = [self.scatter_peak_ratio_p_width, self.scatter_peak_ratio_q_width, self.scale_width] self.b_c_scale_cov_matrix = [[stds[0]**2, self.b_c_corr*stds[0]*stds[1], self.b_scale_corr*stds[0]*stds[2]], [self.b_c_corr*stds[1]*stds[0], stds[1]**2, self.c_scale_corr*stds[1]*stds[2]], [self.b_scale_corr*stds[2]*stds[0], self.c_scale_corr*stds[2]*stds[1], stds[2]**2]] @@ -283,6 +283,8 @@ def InternalConfigure(self, config_dict): self.tritium_model_indices = reader.read_param(config_dict, 'tritium_model_parameters', [0, 2]) self.m_beta_index = self.model_parameter_names.index('m_beta_squared') self.endpoint_index = self.model_parameter_names.index('endpoint') + if self.model_parameter_names: + self.B_index = self.model_parameter_names.index('B') self.fixed_parameters[self.m_beta_index] = not self.fit_nu_mass self.endpoint=reader.read_param(config_dict, 'true_endpoint', 18.573e3) logger.info('Tritium model parameters: {}'.format(np.array(self.model_parameter_names)[self.tritium_model_indices])) @@ -539,14 +541,14 @@ def ConfigureFit(self): #if not self.fit_efficiency_tilt: self.parameter_names = self.model_parameter_names #['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', - # 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', + # 'scatter_peak_ratio_p', 'scatter_peak_ratio_q', # 'res', 'two_gaussia_sigma_1', 'two_gaussian_sigma_2'] self.initial_values = self.model_parameter_means#[self.endpoint, self.background, self.mass_guess**2, self.counts_guess, - #self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, + #self.scatter_peak_ratio_p, self.scatter_peak_ratio_q, #self.res, self.two_gaussian_sigma_1, self.two_gaussian_sigma_2] self.fixes = self.fixed_parameters #[self.fix_endpoint, self.fix_background, self.fix_nu_mass, self.fix_amplitude, - #self.fix_scatter_peak_ratio_b, self.fix_scatter_peak_ratio_c, + #self.fix_scatter_peak_ratio_p, self.fix_scatter_peak_ratio_q, #self.fix_res, self.fix_two_gaussian_sigma_1, self.fix_two_gaussian_sigma_2] self.fixes_dict = self.fixed_parameter_dict @@ -569,8 +571,8 @@ def ConfigureFit(self): # else: # logger.warning('Efficiency tilt will be fitted') # self.tilted_efficiency = True - # self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', 'scatter_peak_ratio_b', 'scatter_peak_ratio_c', 'Efficiency tilt'] - # self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, self.scatter_peak_ratio_b, self.scatter_peak_ratio_c, self.tilt] + # self.parameter_names = ['Endpoint', 'Background', 'm_beta_squared', 'Amplitude', 'scatter_peak_ratio_p', 'scatter_peak_ratio_q', 'Efficiency tilt'] + # self.initial_values = [self.endpoint, 1, self.mass_guess**2, self.counts_guess, self.scatter_peak_ratio_p, self.scatter_peak_ratio_q, self.tilt] # self.parameter_errors = [max([0.1, 0.1*p]) for p in self.initial_values] # self.fixes = [self.fix_endpoint, self.fix_background, self.fix_nu_mass, self.fix_amplitude, self.fix_scatter_ratio_b, self.fix_scatter_ratio_c, self.fix_tilt] # self.limits = [energy_limits, @@ -680,40 +682,40 @@ def SamplePriors(self, sampled_parameters): self.parameter_samples['two_gaussian_sigma_2'] = self.two_gaussian_sigma_2 sample_values.append(self.two_gaussian_sigma_2) if 'scatter_peak_ratio' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio']: - self.scatter_peak_ratio_b = self.Beta_sample(self.scatter_peak_ratio_mean, self.scatter_peak_ratio_width) - self.scatter_peak_ratio_c = 1 + self.scatter_peak_ratio_p = self.Beta_sample(self.scatter_peak_ratio_mean, self.scatter_peak_ratio_width) + self.scatter_peak_ratio_q = 1 self.fix_scatter_ratio_b = True self.fix_scatter_ratio_c = True - self.parameter_samples['scatter_peak_ratio'] = self.scatter_peak_ratio_b - sample_values.append(self.scatter_peak_ratio_b) + self.parameter_samples['scatter_peak_ratio'] = self.scatter_peak_ratio_p + sample_values.append(self.scatter_peak_ratio_p) - if self.correlated_b_c_scale and 'scatter_peak_ratio_b' in sampled_parameters.keys() and 'scatter_peak_ratio_c' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_b'] and sampled_parameters['scatter_peak_ratio_c']: + if self.correlated_b_c_scale and 'scatter_peak_ratio_p' in sampled_parameters.keys() and 'scatter_peak_ratio_q' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_p'] and sampled_parameters['scatter_peak_ratio_q']: logger.info('Correlated b, c, scale sampling') - correlated_vars = np.random.multivariate_normal([self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_c_mean, self.scale_mean], self.b_c_scale_cov_matrix) - self.scatter_peak_ratio_b = correlated_vars[0] - self.scatter_peak_ratio_c = correlated_vars[1] + correlated_vars = np.random.multivariate_normal([self.scatter_peak_ratio_p_mean, self.scatter_peak_ratio_q_mean, self.scale_mean], self.b_c_scale_cov_matrix) + self.scatter_peak_ratio_p = correlated_vars[0] + self.scatter_peak_ratio_q = correlated_vars[1] self.width_scaling = correlated_vars[2] self.fix_scatter_ratio_b = True - self.parameter_samples['scatter_peak_ratio_b'] = self.scatter_peak_ratio_b - sample_values.append(self.scatter_peak_ratio_b) + self.parameter_samples['scatter_peak_ratio_p'] = self.scatter_peak_ratio_p + sample_values.append(self.scatter_peak_ratio_p) - self.parameter_samples['scatter_peak_ratio_c'] = self.scatter_peak_ratio_c - sample_values.append(self.scatter_peak_ratio_c) + self.parameter_samples['scatter_peak_ratio_q'] = self.scatter_peak_ratio_q + sample_values.append(self.scatter_peak_ratio_q) self.fix_scatter_ratio_c = True else: - if 'scatter_peak_ratio_b' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_b']: + if 'scatter_peak_ratio_p' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_p']: logger.info('Uncorrelated b, c, scale sampling') - self.scatter_peak_ratio_b = self.Gamma_sample(self.scatter_peak_ratio_b_mean, self.scatter_peak_ratio_b_width) + self.scatter_peak_ratio_p = self.Gamma_sample(self.scatter_peak_ratio_p_mean, self.scatter_peak_ratio_p_width) self.fix_scatter_ratio_b = True - self.parameter_samples['scatter_peak_ratio_b'] = self.scatter_peak_ratio_b - sample_values.append(self.scatter_peak_ratio_b) - if 'scatter_peak_ratio_c' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_c']: - self.scatter_peak_ratio_c = self.Gamma_sample(self.scatter_peak_ratio_c_mean, self.scatter_peak_ratio_c_width) - self.parameter_samples['scatter_peak_ratio_c'] = self.scatter_peak_ratio_c - sample_values.append(self.scatter_peak_ratio_c) + self.parameter_samples['scatter_peak_ratio_p'] = self.scatter_peak_ratio_p + sample_values.append(self.scatter_peak_ratio_p) + if 'scatter_peak_ratio_q' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_q']: + self.scatter_peak_ratio_q = self.Gamma_sample(self.scatter_peak_ratio_q_mean, self.scatter_peak_ratio_q_width) + self.parameter_samples['scatter_peak_ratio_q'] = self.scatter_peak_ratio_q + sample_values.append(self.scatter_peak_ratio_q) self.fix_scatter_ratio_c = True if 'B' in sampled_parameters.keys() and sampled_parameters['B']: @@ -913,13 +915,8 @@ def gauss_resolution_f(self, energy_array, A, sigma, mu): f = A*(1/(sigma*np.sqrt(2*np.pi)))*np.exp(-(((energy_array-mu)/sigma)**2.)/2.) return f - def beta_rates(self, K, Q, mnu_squared, index): - beta_rates = np.zeros(len(K)) - nu_mass_shape = ((Q - K[index])**2 -mnu_squared)**0.5 - beta_rates[index] = nu_mass_shape*(Q - K[index]) - return beta_rates - def approximate_shape(self, K, Q, m_nu_squared):#, index): + def beta_rates(self, K, Q, m_nu_squared):#, index): spectrum = np.zeros(len(K)) Q_minus_K = Q-K @@ -930,13 +927,13 @@ def approximate_shape(self, K, Q, m_nu_squared):#, index): nu_mass_shape = np.sqrt(nu_mass_shape_squared[index]) spectrum[index] = (Q_minus_K[index])*nu_mass_shape else: - # mainz shape + # mainz shape for negative mbeta**2 k_squared = -m_nu_squared mu = 0.66*np.sqrt(k_squared) index = np.where(Q_minus_K+mu>0) spectrum[index] = (Q_minus_K[index]+mu*np.exp(-1-Q_minus_K[index]/mu))*np.sqrt(Q_minus_K[index]**2+k_squared) #else: - # # lanl + # # lanl shape for negative mbeta**2 # k_squared = -m_nu_squared # index = np.where(Q_minus_K > 0) # spectrum[index] = Q_minus_K[index]**2+k_squared/2 @@ -950,7 +947,7 @@ def chopped_approximate_spectrum(self, E, Q, m_nu_squared): Q_states = Q+self.final_state_array[0]-np.max(self.final_state_array[0]) approximate_e_phase_space = self.ephasespace(E, Q) - beta_rates_array = [self.approximate_shape(E, Q_states[i], m_nu_squared)#, index[i]) + beta_rates_array = [self.beta_rates(E, Q_states[i], m_nu_squared)#, index[i]) * self.final_state_array[1][i] * approximate_e_phase_space for i in range(N_states)] @@ -958,7 +955,7 @@ def chopped_approximate_spectrum(self, E, Q, m_nu_squared): else: approximate_e_phase_space = self.ephasespace(E, Q) - beta_rates_array = self.approximate_shape(E, Q, m_nu_squared) * approximate_e_phase_space + beta_rates_array = self.beta_rates(E, Q, m_nu_squared) * approximate_e_phase_space spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3) * beta_rates_array channel_a_index = np.where((Eself.channel_energy_edges[0][1])) @@ -987,7 +984,7 @@ def approximate_spectrum(self, E, Q, m_nu_squared=0): #index = [np.where(((Q_states[i]-E)**2-m_nu_squared > 0) & (Q_states[i]-E > 0)) for i in range(N_states)] #index = [np.where(E < Q_states[i] -mnu) for i in range(N_states)] - beta_rates_array = [self.approximate_shape(E, Q_states[i], m_nu_squared)#, index[i]) + beta_rates_array = [self.beta_rates(E, Q_states[i], m_nu_squared)#, index[i]) * self.final_state_array[1][i] * approximate_e_phase_space for i in range(N_states)] @@ -995,19 +992,19 @@ def approximate_spectrum(self, E, Q, m_nu_squared=0): else: + approximate_e_phase_space = self.ephasespace(E, Q) + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.beta_rates(E, Q, m_nu_squared)*approximate_e_phase_space - #beta_rates_array = self.approximate_shape(E, Q, m_nu_squared) * approximate_e_phase_space - - + if self.use_relative_livetime_correction: + # scale spectrum in frequency ranges according to channel livetimes + channel_a_index = np.where((Eself.channel_energy_edges[0][1])) + channel_b_index = np.where((Eself.channel_energy_edges[1][1])) + channel_c_index = np.where((Eself.channel_energy_edges[2][1])) - #nu_mass_shape = ((Q - E)**2 -m_nu) - #Q_minus_K = Q-E + spectrum[channel_a_index] = spectrum[channel_a_index]*self.channel_relative_livetimes[0] + spectrum[channel_b_index] = spectrum[channel_b_index]*self.channel_relative_livetimes[1] + spectrum[channel_c_index] = spectrum[channel_c_index]*self.channel_relative_livetimes[2] - #spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3) * beta_rates_array - #index = np.where((nu_mass_shape>0) & (Q_minus_K>0)) - #approximate_e_phase_space = np.zeros(len(E)) - approximate_e_phase_space = self.ephasespace(E, Q) - spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.approximate_shape(E, Q, m_nu_squared)*approximate_e_phase_space return spectrum def ephasespace(self, K, Q): @@ -1022,7 +1019,8 @@ def ephasespace(self, K, Q): def which_model(self, *pars): if self.use_relative_livetime_correction: - return self.chopped_approximate_spectrum(*pars) + return self.approximate_spectrum(*pars) + #return self.chopped_approximate_spectrum(*pars) if self.use_approx_model: return self.approximate_spectrum(*pars) else: @@ -1216,6 +1214,11 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p # tritium args tritium_args = np.array(args)[self.tritium_model_indices] + if 'B' in self.model_parameter_names: + if 'B' in self.parameter_samples.keys() and not self.parameter_samples['B']: + self.B = args[self.B_index] + #self.ReSetBins() + self.ConvertAndHistogram() if len(E)==0: @@ -1291,17 +1294,17 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p if self.is_scattered: # scatter params - if 'scatter_peak_ratio_b' not in self.model_parameter_names: - prob_b = self.scatter_peak_ratio_b - prob_c = self.scatter_peak_ratio_c + if 'scatter_peak_ratio_p' not in self.model_parameter_names: + prob_b = self.scatter_peak_ratio_p + prob_c = self.scatter_peak_ratio_q else: - prob_b = args[self.scatter_peak_ratio_b_index] - prob_c = args[self.scatter_peak_ratio_c_index] + prob_b = args[self.scatter_peak_ratio_p_index] + prob_c = args[self.scatter_peak_ratio_q_index] - if self.fixed_parameters[self.scatter_peak_ratio_b_index]: - prob_b = self.scatter_peak_ratio_b - if self.fixed_parameters[self.scatter_peak_ratio_c_index]: - prob_c = self.scatter_peak_ratio_c + if self.fixed_parameters[self.scatter_peak_ratio_p_index]: + prob_b = self.scatter_peak_ratio_p + if self.fixed_parameters[self.scatter_peak_ratio_q_index]: + prob_c = self.scatter_peak_ratio_q # simplified lineshape FWHM = 2.*np.sqrt(2.*np.log(2.))*res *self.width_scaling From 5e6008807071126b420799ad4321103f98dd85c6 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Fri, 14 Jan 2022 20:23:41 -0800 Subject: [PATCH 153/199] more b and c replaced by p and q --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 8a106a83..9781f478 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -169,7 +169,7 @@ def InternalConfigure(self, config_dict): self.helium_lineshape_path = reader.read_param(config_dict, 'helium_lineshape_path', "optional") self.hydrogen_proportion = reader.read_param(config_dict, 'hydrogen_proportion', 1) self.use_helium_scattering = reader.read_param(config_dict, 'use_helium_scattering', False) - self.correlated_b_c_scale = reader.read_param(config_dict, 'correlated_b_c_scale', False) + self.correlated_p_q_scale = reader.read_param(config_dict, 'correlated_p_q_scale', False) # configure model parameter names @@ -689,7 +689,7 @@ def SamplePriors(self, sampled_parameters): self.parameter_samples['scatter_peak_ratio'] = self.scatter_peak_ratio_p sample_values.append(self.scatter_peak_ratio_p) - if self.correlated_b_c_scale and 'scatter_peak_ratio_p' in sampled_parameters.keys() and 'scatter_peak_ratio_q' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_p'] and sampled_parameters['scatter_peak_ratio_q']: + if self.correlated_p_q_scale and 'scatter_peak_ratio_p' in sampled_parameters.keys() and 'scatter_peak_ratio_q' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_p'] and sampled_parameters['scatter_peak_ratio_q']: logger.info('Correlated b, c, scale sampling') correlated_vars = np.random.multivariate_normal([self.scatter_peak_ratio_p_mean, self.scatter_peak_ratio_q_mean, self.scale_mean], self.b_c_scale_cov_matrix) self.scatter_peak_ratio_p = correlated_vars[0] @@ -925,13 +925,13 @@ def beta_rates(self, K, Q, m_nu_squared):#, index): nu_mass_shape_squared = (Q - K)**2 -m_nu_squared index = np.where((nu_mass_shape_squared>0) & (Q_minus_K>0)) nu_mass_shape = np.sqrt(nu_mass_shape_squared[index]) - spectrum[index] = (Q_minus_K[index])*nu_mass_shape + spectrum[index] = (Q_minus_K[index])*nu_mass_shape*self.ephasespace(K[index], Q) else: # mainz shape for negative mbeta**2 k_squared = -m_nu_squared mu = 0.66*np.sqrt(k_squared) index = np.where(Q_minus_K+mu>0) - spectrum[index] = (Q_minus_K[index]+mu*np.exp(-1-Q_minus_K[index]/mu))*np.sqrt(Q_minus_K[index]**2+k_squared) + spectrum[index] = (Q_minus_K[index]+mu*np.exp(-1-Q_minus_K[index]/mu))*np.sqrt(Q_minus_K[index]**2+k_squared)*self.ephasespace(K[index], Q) #else: # # lanl shape for negative mbeta**2 # k_squared = -m_nu_squared @@ -940,33 +940,33 @@ def beta_rates(self, K, Q, m_nu_squared):#, index): return spectrum - def chopped_approximate_spectrum(self, E, Q, m_nu_squared): + # def chopped_approximate_spectrum(self, E, Q, m_nu_squared): - if self.use_final_states: - N_states = len(self.final_state_array[0]) - Q_states = Q+self.final_state_array[0]-np.max(self.final_state_array[0]) - approximate_e_phase_space = self.ephasespace(E, Q) + # if self.use_final_states: + # N_states = len(self.final_state_array[0]) + # Q_states = Q+self.final_state_array[0]-np.max(self.final_state_array[0]) + # approximate_e_phase_space = self.ephasespace(E, Q) - beta_rates_array = [self.beta_rates(E, Q_states[i], m_nu_squared)#, index[i]) - * self.final_state_array[1][i] - * approximate_e_phase_space for i in range(N_states)] + # beta_rates_array = [self.beta_rates(E, Q_states[i], m_nu_squared)#, index[i]) + # * self.final_state_array[1][i] + # * approximate_e_phase_space for i in range(N_states)] - spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) + # spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) - else: - approximate_e_phase_space = self.ephasespace(E, Q) - beta_rates_array = self.beta_rates(E, Q, m_nu_squared) * approximate_e_phase_space - spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3) * beta_rates_array + # else: + # approximate_e_phase_space = self.ephasespace(E, Q) + # beta_rates_array = self.beta_rates(E, Q, m_nu_squared) * approximate_e_phase_space + # spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3) * beta_rates_array - channel_a_index = np.where((Eself.channel_energy_edges[0][1])) - channel_b_index = np.where((Eself.channel_energy_edges[1][1])) - channel_c_index = np.where((Eself.channel_energy_edges[2][1])) + # channel_a_index = np.where((Eself.channel_energy_edges[0][1])) + # channel_b_index = np.where((Eself.channel_energy_edges[1][1])) + # channel_c_index = np.where((Eself.channel_energy_edges[2][1])) - spectrum[channel_a_index] = spectrum[channel_a_index]*self.channel_relative_livetimes[0] - spectrum[channel_b_index] = spectrum[channel_b_index]*self.channel_relative_livetimes[1] - spectrum[channel_c_index] = spectrum[channel_c_index]*self.channel_relative_livetimes[2] + # spectrum[channel_a_index] = spectrum[channel_a_index]*self.channel_relative_livetimes[0] + # spectrum[channel_b_index] = spectrum[channel_b_index]*self.channel_relative_livetimes[1] + # spectrum[channel_c_index] = spectrum[channel_c_index]*self.channel_relative_livetimes[2] - return spectrum + # return spectrum def approximate_spectrum(self, E, Q, m_nu_squared=0): @@ -993,7 +993,7 @@ def approximate_spectrum(self, E, Q, m_nu_squared=0): else: approximate_e_phase_space = self.ephasespace(E, Q) - spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.beta_rates(E, Q, m_nu_squared)*approximate_e_phase_space + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.beta_rates(E, Q, m_nu_squared)#*approximate_e_phase_space if self.use_relative_livetime_correction: # scale spectrum in frequency ranges according to channel livetimes From 701b9496a035d30dbc65a494fb73c8a1d4875b56 Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Mon, 31 Jan 2022 22:23:51 -0500 Subject: [PATCH 154/199] Fixed freq var approach; added p and q var --- mermithid/misc/FakeTritiumDataFunctions.py | 27 ++++++++++------- .../TritiumSpectrum/FakeDataGenerator.py | 29 +++++++++++++++---- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 3c4bade1..59ac0913 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -278,7 +278,7 @@ def convolved_bkgd_rate(K, Kmin, Kmax, lineshape, ls_params, min_energy, max_ene #Convolution of signal and lineshape using scipy.signal.convolve def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape, ls_params, scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, min_energy, max_energy, - complexLineShape, final_state_array, resolution_function, ins_res_width_bounds, ins_res_width_factors): + complexLineShape, final_state_array, resolution_function, ins_res_width_bounds, ins_res_width_factors, p_factors, q_factors): """K is an array-like object """ logger.info('Using scipy convolve') @@ -308,32 +308,39 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, if resolution_function == 'simulated_resolution' or resolution_function == 'simulated': lineshape_rates = [] scale_factors = [ls_params[0]*f for f in ins_res_width_factors] - for scale in scale_factors: - lineshape_rates.append(np.flipud(complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale, ls_params[1], scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, emitted_peak='dirac'))) + for i in range(len(scale_factors)): + lineshape_rates.append(np.flipud(complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale_factors[i], ls_params[1], scatter_peak_ratio_p*p_factors[i], scatter_peak_ratio_q*q_factors[i], scatter_fraction, emitted_peak='dirac'))) elif resolution_function == 'gaussian_resolution' or resolution_function == 'gaussian': logger.warn("Scatter peak ratio function for lineshape with Gaussian resolution may not be up-to-date!") lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, emitted_peak='dirac') else: logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) + below_Kmin = np.where(K < Kmin) + #Convolving if (lineshape=='detailed_scattering' or lineshape=='detailed') and (resolution_function == 'simulated_resolution' or resolution_function == 'simulated'): - convolved_list = [] + convolved_segments = [] + beta_rates = spectral_rate(K, Q, mnu, final_state_array) for j in range(len(lineshape_rates)): - beta_rates = spectral_rate(K_segments[j], Q, mnu, final_state_array) - convolved_list.append(convolve(beta_rates, lineshape_rates[j], mode='same')) - convolved = np.concatenate(convolved_list, axis=None) + #beta_rates = spectral_rate(K_segments[j], Q, mnu, final_state_array) + convolved_j = convolve(beta_rates, lineshape_rates[j], mode='same') + np.put(convolved_j, below_Kmin, np.zeros(len(below_Kmin))) + #Only including the part of convolved_j that corresponds to the right values of K + convolved_segments.append(convolved_j[np.logical_and(Kbounds[j]<=K, K<=Kbounds[j+1])]) + #convolved.append(convolved_j) + convolved = np.concatenate(convolved_segments, axis=None) elif resolution_function=='gaussian': lineshape_rates = np.flipud(lineshape_rates) beta_rates = spectral_rate(K, Q, mnu, final_state_array) convolved = convolve(beta_rates, lineshape_rates, mode='same') + np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) if (lineshape=='gaussian' or lineshape=='simplified_scattering' or lineshape=='simplified'): beta_rates = spectral_rate(K, Q, mnu, final_state_array) convolved = convolve(beta_rates, lineshape_rates, mode='same') - - below_Kmin = np.where(K < Kmin) - np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) + np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) + return convolved diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 21ea6641..138f9ec2 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -146,7 +146,9 @@ def InternalConfigure(self, params): self.min_energy = reader.read_param(params,'min_lineshape_energy', -1000) self.scale_factor = reader.read_param(params, 'scale_factor', 1.0) self.ins_res_width_bounds = reader.read_param(params, 'ins_res_width_bounds', None) #Default values here need to be corrected - self.ins_res_width_factors = reader.read_param(params, 'ins_res_width_factors', [1]) + self.ins_res_width_factors = reader.read_param(params, 'ins_res_width_factors', [1.]) + self.p_factors = reader.read_param(params, 'p_factors', [1.]) + self.q_factors = reader.read_param(params, 'q_factors', [1.]) #paths self.efficiency_path = reader.read_param(params, 'efficiency_path', '') @@ -390,14 +392,17 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 if array_method == True: ratesS = convolved_spectral_rate_arrays(self.Koptions, Q_mean, mass, Kmin, lineshape, params, self.scatter_peak_ratio_p, self.scatter_peak_ratio_q, self.scatter_proportion, min_energy, max_energy, - self.complexLineShape, self.final_state_array, self.resolution_function, self.ins_res_width_bounds, self.ins_res_width_factors) + self.complexLineShape, self.final_state_array, self.resolution_function, self.ins_res_width_bounds, self.ins_res_width_factors, self.p_factors, self.q_factors) else: ratesS = [convolved_spectral_rate(K, Q_mean, mass, Kmin, lineshape, params, min_energy, max_energy) for K in self.Koptions] # multiply rates by efficiency + #if self.ins_res_width_bounds==None: ratesS = ratesS*efficiency + #else: + #ratesS = [r*efficiency for r in ratesS] time1 = time.time() logger.info('... signal rate took {} s'.format(time1-time0)) @@ -423,13 +428,24 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 #Generating finely spaced points on a gaussian gaussian_rates = gaussian(K_lineshape, [err_from_B, 0]) - ratesS = convolve(ratesS, gaussian_rates, mode='same') ratesB = convolve(ratesB, gaussian_rates, mode='same') - + #if self.ins_res_width_bounds==None: + ratesS = convolve(ratesS, gaussian_rates, mode='same') + ratesS[ratesS<0.] = 0. - ratesB[ratesB<0.] = 0. - rate_sumS, rate_sumB = np.sum(ratesS), np.sum(ratesB) + rate_sumS = np.sum(ratesS) probsS = np.array(ratesS)/rate_sumS + """ + else: + rate_sumS, probsS = [], [] + for i in range(len(ratesS)): + ratesS[i] = convolve(ratesS[i], gaussian_rates, mode='same') + rate_sumS.append(np.sum(ratesS[i])) + probsS.append(np.array(ratesS[i])/rate_sumS[i]) + """ + + ratesB[ratesB<0.] = 0. + rate_sumB = np.sum(ratesB) probsB = np.array(ratesB)/rate_sumB #Calculate three different rates variables, for each of the three runtimes @@ -441,6 +457,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 #Break up self.Koptions into three different arrays. #Then, sample KE variables for each of the arrays and appropriate elements of self.channel_runtimes, probsS and probsB. #Finally, concatenate together the three KE arrays. + #if self.ins_res_width_bounds==None: temp_Koptions, temp_probsS, temp_probsB = self.Koptions, probsS, probsB split_Koptions, split_probsS, split_probsB = [], [], [] for i in range(len(self.channel_bounds)): From 749786f9973cfcfca0fb8d32807a472b1f2f39f0 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 9 Feb 2022 10:15:37 -0800 Subject: [PATCH 155/199] e phase space now calculated in beta_rates --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 9781f478..fe8a462a 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -55,7 +55,7 @@ def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, logger.info('Sampling: {}'.format(i)) - T = BinnedTritiumMLFitter("TritiumFitter") + T = BinnedTritiumMLFitter("TritiumFitter_{}".format(i)) T.InternalConfigure(fit_config_dict) #T.error_scaling = error_scaling T.integrate_bins = True @@ -80,7 +80,7 @@ def GenAndFit(params, counts, fit_config_dict, sampled_priors={}, results, errors = T.SampleConvertAndFit(sampled_priors, params) parameter_samples = T.parameter_samples - logger.info('Fit results: {}'.format(results)) + #logger.info('Fit results: {}'.format(results)) if fit_config_dict['minos_intervals']: @@ -980,20 +980,19 @@ def approximate_spectrum(self, E, Q, m_nu_squared=0): if self.use_final_states: N_states = len(self.final_state_array[0]) Q_states = Q+self.final_state_array[0]-np.max(self.final_state_array[0]) - approximate_e_phase_space = self.ephasespace(E, Q) #index = [np.where(((Q_states[i]-E)**2-m_nu_squared > 0) & (Q_states[i]-E > 0)) for i in range(N_states)] #index = [np.where(E < Q_states[i] -mnu) for i in range(N_states)] beta_rates_array = [self.beta_rates(E, Q_states[i], m_nu_squared)#, index[i]) * self.final_state_array[1][i] - * approximate_e_phase_space for i in range(N_states)] + for i in range(N_states)] spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) else: - approximate_e_phase_space = self.ephasespace(E, Q) - spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.beta_rates(E, Q, m_nu_squared)#*approximate_e_phase_space + #approximate_e_phase_space = self.ephasespace(E, Q) + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.beta_rates(E, Q, m_nu_squared) if self.use_relative_livetime_correction: # scale spectrum in frequency ranges according to channel livetimes From 2cd0df7450795ba3ed1398c3b6cb62cb1f2dfc53 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 10 Feb 2022 09:25:38 -0800 Subject: [PATCH 156/199] linking two gaussian model parameters to resolution --- .../processors/Fitters/BinnedDataFitter.py | 2 +- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 79 +++++++++++++------ 2 files changed, 55 insertions(+), 26 deletions(-) diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index 53866d99..d500a691 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -143,7 +143,7 @@ def fit(self): # minimze m_binned.simplex().migrad() - m_binned.hesse() + #m_binned.hesse() #m_binned.minos() #self.param_states = m_binned.get_param_states() #logger.info(self.param_states) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index fe8a462a..c304f65b 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -170,15 +170,16 @@ def InternalConfigure(self, config_dict): self.hydrogen_proportion = reader.read_param(config_dict, 'hydrogen_proportion', 1) self.use_helium_scattering = reader.read_param(config_dict, 'use_helium_scattering', False) self.correlated_p_q_scale = reader.read_param(config_dict, 'correlated_p_q_scale', False) + self.derived_two_gaussian_model = reader.read_param(config_dict, 'derived_two_gaussian_model', True) # configure model parameter names self.model_parameter_names = reader.read_param(config_dict, 'model_parameter_names', ['endpoint', 'background', 'm_beta_squared', 'Amplitude', 'scatter_peak_ratio_p', 'scatter_peak_ratio_q', - 'resolution', 'two_gaussian_sigma_1', 'two_gaussian_sigma_2'] ) + 'resolution', 'two_gaussian_mu_1', 'two_gaussian_mu_2'] ) # initial values and mean of constaints (if constraint) or mean of distribution (if sampled) - self.model_parameter_means = reader.read_param(config_dict, 'model_parameter_means', [18.6e3, 0, 0, 5000, 0.8, 1, 15, 10, 10]) + self.model_parameter_means = reader.read_param(config_dict, 'model_parameter_means', [18.6e3, 0, 0, 5000, 0.8, 1, 15, 0, 0]) # width for constraints or sample distributions self.model_parameter_widths = reader.read_param(config_dict, 'model_parameter_widths', [100, 0.1, 0.1, 500, 0.1, 0, 3, 1, 1]) self.fixed_parameters = reader.read_param(config_dict, 'fixed_parameters', [False, False, False, False, True, True, True, True, True]) @@ -191,8 +192,8 @@ def InternalConfigure(self, config_dict): [0.1, 1.], [0.1, 1.], [12, 100], - [1, 100], - [1, 100]]) + [-100, 100], + [-100, 100]]) # check that configuration is consistent @@ -230,20 +231,33 @@ def InternalConfigure(self, config_dict): self.res = self.res_mean if self.resolution_model != 'gaussian': - if 'two_gaussian_sigma_1' in self.model_parameter_names: - self.two_gaussian_sigma_1_index = self.model_parameter_names.index('two_gaussian_sigma_1') - if 'two_gaussian_sigma_2' in self.model_parameter_names: - self.two_gaussian_sigma_2_index = self.model_parameter_names.index('two_gaussian_sigma_2') - + self.two_gaussian_fraction = reader.read_param(config_dict, 'two_gaussian_fraction', 1.) self.two_gaussian_mu_1 = reader.read_param(config_dict, 'two_gaussian_mu1', 0) self.two_gaussian_mu_2 = reader.read_param(config_dict, 'two_gaussian_mu2', 0) - self.two_gaussian_sigma_1_mean = reader.read_param(config_dict, 'two_gaussian_sigma_1_mean', 15) - self.two_gaussian_sigma_2_mean = reader.read_param(config_dict, 'two_gaussian_sigma_2_mean', 5) - self.two_gaussian_sigma_1_width = reader.read_param(config_dict, 'two_gaussian_sigma_1_width', 1) - self.two_gaussian_sigma_2_width = reader.read_param(config_dict, 'two_gaussian_sigma_2_width', 1) - self.two_gaussian_wide_fraction = reader.read_param(config_dict, 'two_gaussian_wide_fraction', 1.) - self.two_gaussian_sigma_1 = deepcopy(self.two_gaussian_sigma_1_mean) - self.two_gaussian_sigma_2 = deepcopy(self.two_gaussian_sigma_2_mean) + if 'two_gaussian_mu_1' in self.model_parameter_names: + self.two_gaussian_mu_1_index = self.model_parameter_names.index('two_gaussian_mu_1') + if 'two_gaussian_mu_2' in self.model_parameter_names: + self.two_gaussian_mu_2_index = self.model_parameter_names.index('two_gaussian_mu_2') + + if self.derived_two_gaussian_model: + self.two_gaussian_p0 = reader.read_param(config_dict, 'two_gaussian_p0', 1.) + self.two_gaussian_p1 = reader.read_param(config_dict, 'two_gaussian_p1', 1.) + else: + if 'two_gaussian_sigma_1' in self.model_parameter_names: + self.two_gaussian_sigma_1_index = self.model_parameter_names.index('two_gaussian_sigma_1') + if 'two_gaussian_sigma_2' in self.model_parameter_names: + self.two_gaussian_sigma_2_index = self.model_parameter_names.index('two_gaussian_sigma_2') + + + self.two_gaussian_sigma_1_mean = reader.read_param(config_dict, 'two_gaussian_sigma_1_mean', 15) + self.two_gaussian_sigma_2_mean = reader.read_param(config_dict, 'two_gaussian_sigma_2_mean', 5) + self.two_gaussian_sigma_1_width = reader.read_param(config_dict, 'two_gaussian_sigma_1_width', 1) + self.two_gaussian_sigma_2_width = reader.read_param(config_dict, 'two_gaussian_sigma_2_width', 1) + # initial setting + self.two_gaussian_sigma_1 = deepcopy(self.two_gaussian_sigma_1_mean) + self.two_gaussian_sigma_2 = deepcopy(self.two_gaussian_sigma_2_mean) + + # scatter peak ratio @@ -915,6 +929,13 @@ def gauss_resolution_f(self, energy_array, A, sigma, mu): f = A*(1/(sigma*np.sqrt(2*np.pi)))*np.exp(-(((energy_array-mu)/sigma)**2.)/2.) return f + def derived_two_gaussian_resolution(self, energy_array, sigma_s, mu_1, mu_2, A=1): + sigma_1 = (sigma_s-self.two_gaussian_p0 + self.two_gaussian_fraction * self.two_gaussian_p0)/(self.two_gaussian_fraction + self.two_gaussian_p1 - self.two_gaussian_fraction* self.two_gaussian_p1) + sigma_2 = self.two_gaussian_p0 + self.two_gaussian_p1 * sigma_1 + + lineshape = self.two_gaussian_fraction * self.gauss_resolution_f(energy_array, 1, sigma_1*self.width_scaling, mu_1) + (1 - self.two_gaussian_fraction) * self.gauss_resolution_f(energy_array, 1, sigma_2*self.width_scaling, mu_2) + return lineshape + def beta_rates(self, K, Q, m_nu_squared):#, index): spectrum = np.zeros(len(K)) @@ -1249,17 +1270,23 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p res = args[self.res_index] if self.resolution_model != 'gaussian': + if self.derived_two_gaussian_model: + if 'two_gaussian_mean_1' not in self.model_parameter_names or 'two_gaussian_mean_1' in self.parameter_samples.keys(): + two_gaussian_mu_1 = self.two_gaussian_mu_1 + two_gaussian_mu_2 = self.two_gaussian_mu_2 + else: + two_gaussian_mu_1 = args[self.two_gaussian_mu_1_index] + two_gaussian_mu_2 = args[self.two_gaussian_mu_2_index] - if 'two_gaussian_sigma_1' not in self.model_parameter_names or 'two_gaussian_sigma_1' in self.parameter_samples.keys(): - sig1 = self.two_gaussian_sigma_1 - #logger.info('Using self.two_gaussian_sigma_1') else: - sig1 = args[self.two_gaussian_sigma_1_index] - if 'two_gaussian_sigma_2' not in self.model_parameter_names or 'two_gaussian_sigma_2' in self.parameter_samples.keys(): - sig2 = self.two_gaussian_sigma_2 - #logger.info('Using self.two_gaussian_sigma_2') - else: - sig2 = args[self.two_gaussian_sigma_2_index] + + if 'two_gaussian_sigma_1' not in self.model_parameter_names or 'two_gaussian_sigma_1' in self.parameter_samples.keys(): + sig1 = self.two_gaussian_sigma_1 + sig2 = self.two_gaussian_sigma_2 + #logger.info('Using self.two_gaussian_sigma_1') + else: + sig1 = args[self.two_gaussian_sigma_1_index] + sig2 = args[self.two_gaussian_sigma_2_index] max_energy = self.max_energy @@ -1275,6 +1302,8 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p # energy resolution if self.resolution_model != 'two_gaussian': lineshape = self.gauss_resolution_f(e_lineshape, 1, res*self.width_scaling, 0) + elif self.derived_two_gaussian_model: + lineshape = self.derived_two_gaussian_resolution(e_lineshape, res, two_gaussian_mu_1, two_gaussian_mu_2) else: lineshape = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, sig1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, sig2*self.width_scaling, self.two_gaussian_mu_2) From f4394ab4520e1e3c91f51db96a2aee72c5264573 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 2 Mar 2022 18:59:45 -0800 Subject: [PATCH 157/199] fixes to using derived two-gaussian model --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index c304f65b..2687e42f 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -297,7 +297,7 @@ def InternalConfigure(self, config_dict): self.tritium_model_indices = reader.read_param(config_dict, 'tritium_model_parameters', [0, 2]) self.m_beta_index = self.model_parameter_names.index('m_beta_squared') self.endpoint_index = self.model_parameter_names.index('endpoint') - if self.model_parameter_names: + if 'B' in self.model_parameter_names: self.B_index = self.model_parameter_names.index('B') self.fixed_parameters[self.m_beta_index] = not self.fit_nu_mass self.endpoint=reader.read_param(config_dict, 'true_endpoint', 18.573e3) @@ -1374,12 +1374,13 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p #plt.plot(e_lineshape, self.simplified_ls(e_lineshape, 0, FWHM, ratio), color='red', label='FDG') if self.resolution_model != 'two_gaussian': resolution = self.gauss_resolution_f(e_lineshape, 1, res, 0) - if self.plot_lineshape: - logger.info('Using Gaussian resolution model') + logger.info('Using Gaussian resolution model') + elif self.derived_two_gaussian_model: + resolution = self.derived_two_gaussian_resolution(e_lineshape, res, two_gaussian_mu_1, two_gaussian_mu_2) + logger.info('Using derived two Gaussian resolution model') else: - resolution = self.two_gaussian_wide_fraction * self.gauss_resolution_f(e_lineshape, 1, sig1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_wide_fraction) * self.gauss_resolution_f(e_lineshape, 1, sig2*self.width_scaling, self.two_gaussian_mu_2) - if self.plot_lineshape: - logger.info('Using two Gaussian resolution model') + resolution = self.two_gaussian_fraction * self.gauss_resolution_f(e_lineshape, 1, sig1*self.width_scaling, self.two_gaussian_mu_1) + (1 - self.two_gaussian_fraction) * self.gauss_resolution_f(e_lineshape, 1, sig2*self.width_scaling, self.two_gaussian_mu_2) + logger.info('Using two Gaussian resolution model') plt.plot(e_lineshape, resolution/np.max(resolution), label = 'Resolution', color='orange') From 433f2fd5842f943274945d69d8b1321ad0591609 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 6 Apr 2022 13:40:29 -0700 Subject: [PATCH 158/199] hydrogen fraction is now fit parameter --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 2687e42f..2b39f68d 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -266,6 +266,8 @@ def InternalConfigure(self, config_dict): self.scatter_peak_ratio_p_index = self.model_parameter_names.index('scatter_peak_ratio_p') if 'scatter_peak_ratio_q' in self.model_parameter_names: self.scatter_peak_ratio_q_index = self.model_parameter_names.index('scatter_peak_ratio_q') + if 'h2_fraction' in self.model_parameter_names: + self.h2_fraction_index = self.model_parameter_names.index('h2_fraction') self.spr_factor = reader.read_param(config_dict, 'SPR_factor', 0) self.scatter_peak_ratio_p_mean = reader.read_param(config_dict, 'scatter_peak_ratio_p_mean', 0.7) @@ -1156,7 +1158,7 @@ def more_accurate_simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, return lineshape, norm - def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): + def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1, h2_fraction=1): """ This uses Gaussians of different mu and sigma for different gases """ @@ -1208,7 +1210,8 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): #plt.plot(K, (shape + hydrogen_scattering)/np.max(shape + hydrogen_scattering), color='blue', label='hydrogen') #plt.plot(K, (shape + helium_scattering)/np.max(shape + helium_scattering), color='red', label='helium') # full lineshape - lineshape = self.hydrogen_proportion*hydrogen_scattering + (1-self.hydrogen_proportion)*helium_scattering + #lineshape = self.hydrogen_proportion*hydrogen_scattering + (1-self.hydrogen_proportion)*helium_scattering + lineshape = h2_fraction*hydrogen_scattering + (1-h2_fraction)*helium_scattering #plt.plot(K, lineshape/np.max(lineshape), color='black', label='full') #plt.xlim(-200, 200) @@ -1334,6 +1337,11 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p if self.fixed_parameters[self.scatter_peak_ratio_q_index]: prob_c = self.scatter_peak_ratio_q + if 'h2_fraction' in self.model_parameter_names: + h2_fraction = args[self.h2_fraction_index] + else: + h2_fraction = self.hydrogen_proportion + # simplified lineshape FWHM = 2.*np.sqrt(2.*np.log(2.))*res *self.width_scaling @@ -1343,7 +1351,7 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p if self.plot_lineshape: logger.info('Using simplified lineshape model') else: - tail, norm = self.multi_gas_lineshape(e_lineshape, 0, FWHM, prob_b, prob_c) + tail, norm = self.multi_gas_lineshape(e_lineshape, 0, FWHM, prob_b, prob_c, h2_fraction) if self.plot_lineshape: logger.info('Using two gas simplified lineshape model') From 70f2c50a4fc637f144b6eecc60bfa0d8d86f46b4 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Mon, 18 Apr 2022 13:14:33 -0700 Subject: [PATCH 159/199] added frequency dependent detector response and made frequentists analysis runable in the mermithid container --- Dockerfile | 2 +- .../misc/DetectionEfficiencyUtilities.py | 182 +++++++++++++++++- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 160 ++++++++++++--- 3 files changed, 313 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index de84a0ae..e0e8c544 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,10 +55,10 @@ RUN source $MERMITHID_BUILD_PREFIX/setup.sh &&\ cd /tmp_source &&\ # ls -altrh morpho &&\ pip3 install . ./morpho --prefix $MERMITHID_BUILD_PREFIX &&\ - pip3 install iminuit &&\ /bin/true ######################## FROM mermithid_common COPY --from=mermithid_done $MERMITHID_BUILD_PREFIX $MERMITHID_BUILD_PREFIX +RUN pip3 install seaborn statsmodels diff --git a/mermithid/misc/DetectionEfficiencyUtilities.py b/mermithid/misc/DetectionEfficiencyUtilities.py index 475d363d..7c04e84d 100755 --- a/mermithid/misc/DetectionEfficiencyUtilities.py +++ b/mermithid/misc/DetectionEfficiencyUtilities.py @@ -6,8 +6,33 @@ from scipy.interpolate import interp1d from scipy.stats import binom from scipy import constants -import helper_functions.alis_power as ap -import helper_functions.plot_methods as pm + +from scipy import constants +import scipy.special as scs + +#import moviepy.editor as mpy +c = 299792458 +m_kg = 9.10938291*1e-31 +me_keV = 510.998 +q_C = 1.60217657*1e-19 +Z0 = 119.917 * np.pi +Rc = 0.06e-2 +L0 = 0.35 +L0 = 0.3 +L0 = 0.2 +H = 0.9359 +H = 0.95777194923080811 +H = 0.9578170819250281 +E = 17.83 +a = 1.07e-2 +b = 0.43e-2 +r = 1.006e-2/2 #Circ WG Radius +theta = np.arange( 88 * np.pi / 180 , 89.999 * np.pi / 180 , 0.0001) +#theta = 89.9*np.pi/180 + +# ?? +n = 0 + def binomial_interval(n, k, alpha=1 / 3.1514872): @@ -72,7 +97,7 @@ def power_efficiency(f, plot=False, savepath='.'): N = int(1e6) efficiency = np.zeros(len(f)) efficiency_error = np.zeros((len(f),2)) - power_factor = ap.Power(f)/ap.Power([1407e6+24.5e9]) + power_factor = Power(f)/Power([1407e6+24.5e9]) #if plot: # N = int(1e6) @@ -81,8 +106,8 @@ def power_efficiency(f, plot=False, savepath='.'): if plot: plt.figure(figsize=(7,5)) - plt.hist(data0, histtype='step', bins=100, color=pm.sns_color[0]) - plt.axvline(threshold, label='Detection threshold', color=pm.sns_color[4]) + plt.hist(data0, histtype='step', bins=100, color='blue') + plt.axvline(threshold, label='Detection threshold', color='red') plt.ylabel('N') plt.legend() plt.xlabel('SNR') @@ -257,3 +282,150 @@ def pseudo_integrated_efficiency(f, df, snr_efficiency_dict, alpha=1): return pseudo_y+y, y_error + +def sinc(x): + if x == 0 : + return 1 + return np.sin(x)/x +def cot(x): + return np.cos(x)/np.sin(x) + +def alpha_function(n, L0, E, H, theta ): + gamma = 1 + E / me_keV + wc = q_C * H / ( gamma * m_kg ) + v0 = c * ( 1 - 1 / gamma ** 2 )**0.5 + wa = v0 / L0 * np.sin(theta) + #wa = v0 / L0 * np.mean(np.sin(theta)) + f = wc / ( 2 * np.pi ) + q = -0.25 * wc/wa * cot(theta)**2 + #q = -0.25 * wc/wa * np.mean(cot(theta)**2) + + return scs.jv( n , q ) + +def beta_function(n, L0, E, H, theta ): + + gamma = 1 + E / me_keV + wc = q_C * H / ( gamma * m_kg ) + v0 = c * ( 1 - 1 / gamma ** 2 )**0.5 + vz0 = v0 * np.cos(theta) + #vz0 = v0 * np.mean(np.cos(theta)) + wa = v0 / L0 * np.sin(theta) + #wa = v0 / L0 * np.mean(np.sin(theta)) + f = wc / ( 2 * np.pi ) + Dw = 0.5 * wc * cot(theta)**2 + #Dw = 0.5 * wc * np.mean(cot(theta)**2) + fc=c/(2*a) + vp = c/(1-(2*np.pi*fc/(wc+Dw))**2)**0.5 + k = (wc + Dw) / vp + zmax = L0 * cot( theta ) + #zmax = L0 * np.mean(cot( theta )) + + return scs.jv( n , k * zmax ) + +def freq_func( n, L0, E, H, theta ): + gamma = 1 + E / me_keV + wc = q_C * H / ( gamma * m_kg ) + v0 = c * ( 1 - 1 / gamma ** 2 )**0.5 + wa = v0 / L0 * np.sin(theta) + return (wc+n*wa)/(2*np.pi) + +#def power_function(n, L0, E, H, theta ): +# +# an = 0 +# for m in range( -5, 5 ): +# an += alpha_function ( n = m, L0 = L0 , E = E , H = H, theta = theta ) * beta_function( n = n - 2 * m, L0 = L0 , E = E , H = H, theta = theta ) +# return abs( an ) ** 2 + +def power_function(n, L0, E, H, theta): + zmax = L0 / np.tan(theta/180*np.pi) + gamma = 1 + E / me_keV + wc = q_C * H / ( gamma * m_kg ) + Dw = 0.5 * wc * cot(theta)**2 + fc=c/(2*a) + vp = c/(1-(2*np.pi*fc/(wc+Dw))**2)**0.5 + k = wc / c + return scs.jv(n, k*zmax)**2 + +def start_freq_func( L0, E, H, theta ): + gamma = 1 + E / me_keV + wc = q_C * H / ( gamma * m_kg ) + Dw = 0.5 * wc * cot(theta)**2 + #Dw = 0.5 * wc * np.mean(cot(theta)**2) + #print(Dw) + return ( wc + Dw ) / ( 2 * np.pi ) + +#def start_freq_func( L0, E, H, theta ): +# gamma = 1 + E / me_keV +# wc = q_C * H / ( gamma * m_kg ) +# Dw = 0.5 * wc * cot(theta)**2 +# #print Dw +# return ( wc + Dw ) / ( 2 * np.pi ) + +fc11 = 1.841 * c / (2 * np.pi * r) + + +def z_11(f): + return Z0 / (1 - (fc11/f)**2)**0.5 + +def p_11(f, E, rho): + gamma = 1 + E / me_keV + v0 = c * ( 1 - 1 / gamma ** 2 )**0.5 + + alpha = 0.108858 * r**2 + return z_11(f) * q_C**2 * v0**2 / (8 * np.pi * alpha) * ( scs.jvp(1, 1.841 * rho / r )**2 + 1 / (1.841 * rho / r )**2 * scs.j1(1.841 * rho / r)**2 ) + + +def Power_17keV(f): + f = np.array(f) + P = np.zeros(len(f)) + + # powers from ali + E = 17.826e3 + r = 0.0001 + n = 0 + + for i in range(len(f)): + h = B_field(E*1e-3, f[i]) + bla = start_freq_func(L0, E*1e-3, h, np.pi/2) + p = (p_11(bla, E*1e-3, r)) + + P[i] = p + + return P + + +def Power(f): + # powers from ali + f = np.array(f) + P = np.zeros(len(f)) + + E = energy(f) + r = 0.0001 + n = 0 + + + for i in range(len(E)): + p_dist = power_function(n, L0, E[i]*1e-3, H, np.pi/2) + bla = (start_freq_func(L0, E[i]*1e-3, H, np.pi/2)) + p = p_11(bla, E[i]*1e-3, r) + + P[i] = p + + return P + + +def energy(f, mixfreq=0): + + B = 0.9578170819250281 + emass = constants.electron_mass/constants.e*constants.c**2 + gamma = (constants.e*B)/(2.0*np.pi*constants.electron_mass) * 1/(f+mixfreq) + #shallow trap 0.959012745568 + #deep trap 0.95777194923080811 + return (gamma -1)*emass + + + +def B_field(e_keV, f): + emass = constants.electron_mass/constants.e*constants.c**2*1e-3 + b = (e_keV*1.0/emass + 1) * 2*np.pi*constants.electron_mass * f / constants.e + return b diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 2b39f68d..4b178ac6 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -23,6 +23,8 @@ from scipy.special import erfc import matplotlib.pyplot as plt +from scipy.interpolate import interp1d + from morpho.utilities import morphologging, reader logger = morphologging.getLogger(__name__) @@ -161,6 +163,9 @@ def InternalConfigure(self, config_dict): self.fit_nu_mass = reader.read_param(config_dict, 'fit_neutrino_mass', False) self.use_relative_livetime_correction = reader.read_param(config_dict, 'use_relative_livetime_correction', False) + # save plots in (processor can plot lineshape used in tritium model) + self.savepath = reader.read_param(config_dict, 'savepath', '.') + # detector response options self.NScatters = reader.read_param(config_dict, 'NScatters', 20) self.resolution_model = reader.read_param(config_dict, 'resolution_model', 'gaussian') @@ -169,6 +174,7 @@ def InternalConfigure(self, config_dict): self.helium_lineshape_path = reader.read_param(config_dict, 'helium_lineshape_path', "optional") self.hydrogen_proportion = reader.read_param(config_dict, 'hydrogen_proportion', 1) self.use_helium_scattering = reader.read_param(config_dict, 'use_helium_scattering', False) + self.use_frequency_dependent_lineshape = reader.read_param(config_dict, 'use_frequency_dependent_lineshape', True) self.correlated_p_q_scale = reader.read_param(config_dict, 'correlated_p_q_scale', False) self.derived_two_gaussian_model = reader.read_param(config_dict, 'derived_two_gaussian_model', True) @@ -310,9 +316,19 @@ def InternalConfigure(self, config_dict): self.use_final_states = reader.read_param(config_dict, 'use_final_states', False) self.final_state_array = reader.read_param(config_dict, 'final_states_array', [[0], [1]]) + #self.max_final_state_energy_loss = np.max(np.abs(self.final_state_array[0])) + #self.final_states_interp = interp1d(self.final_state_array[0]-np.max(self.final_state_array[0]), self.final_state_array[1], bounds_error=False, fill_value=0) + #max_energy = self.max_final_state_energy_loss + #dE = 1 + #n_dE = round(max_energy/dE) + #e_lineshape = np.arange(-n_dE*dE, n_dE*dE, dE) + #plt.figure() + #plt.plot(e_lineshape, self.final_states_interp(e_lineshape)) + #plt.yscale('log') + #plt.savefig(os.path.join(self.savepath,'final_states_interp.png'), dpi=300) + + - # save plots in (processor can plot lineshape used in tritium model) - self.savepath = reader.read_param(config_dict, 'savepath', '.') # individual model parameter configurations @@ -327,7 +343,7 @@ def InternalConfigure(self, config_dict): # frequency an energy range self.min_frequency = reader.read_param(config_dict, 'min_frequency', "required") self.max_frequency = reader.read_param(config_dict, 'max_frequency', None) - self.max_energy = reader.read_param(config_dict, 'resolution_energy_max', 1200) + self.max_energy = reader.read_param(config_dict, 'resolution_energy_max', 1000) # path to json with efficiency dictionary self.efficiency_file_path = reader.read_param(config_dict, 'efficiency_file_path', '') @@ -339,6 +355,16 @@ def InternalConfigure(self, config_dict): self.channel_livetimes = reader.read_param(config_dict, 'channel_livetimes', [7185228, 7129663, 7160533]) self.channel_relative_livetimes = np.array(self.channel_livetimes)/ np.max(self.channel_livetimes) + # freqeuncy dependent detector response + self.ins_res_bounds_freq = reader.read_param(config_dict, 'ins_res_bounds_freq', []) + self.scatter_factor_bounds = self.Energy(self.ins_res_bounds_freq) + self.p_factors = np.array(reader.read_param(config_dict, 'p_factors', [1])) + self.q_factors = np.array(reader.read_param(config_dict, 'q_factors', [1])) + self.width_factors = np.array(reader.read_param(config_dict, 'res_width_factors', [1])) + + #self.scatter_factor_bounds = np.sort(self.Energy([25.90*10**9, 25.91*10**9, 25.92*10**9, 25.925*10**9, 25.93*10**9, 25.94*10**9, 25.96*10**9, 25.965*10**9, 25.9675*10**9, 25.968*10**9, 25.97*10**9, 25.98*10**9])) + #self.p_factors = np.flip([0.9983968696582269, 1.0241234738781622, 1.074069218435362, 1.1880615082443595, 0.7630526168402478, 0.9252229133675792, 1.0099495204807443, 1.0305682175456141, 1.754449151085981, 1.0427487909986666, 1.0074967873980396, 0.8585656489155274, 0.9830120037530994]) + #self.q_factors = np.flip([0.9961848753897812, 0.9911348282030333, 1.003671359578592, 0.9575276167823439, 1.022755464502686, 1.0139235105836255, 0.9964227505579599, 0.9854455059345174, 0.9613799082228937, 1.0068195698448816, 0.9995181762199474, 1.0273166210271425, 1.0054718471162767]) @@ -761,7 +787,7 @@ def Energy(self, f, mixfreq=0.): converts frequency in Hz to energy in eV """ emass = constants.electron_mass/constants.e*constants.c**2 - gamma = (constants.e*self.B)/(2.0*np.pi*constants.electron_mass) * 1./(f+mixfreq) + gamma = (constants.e*self.B)/(2.0*np.pi*constants.electron_mass) * 1./(np.array(f)+mixfreq) return (gamma -1.)*emass @@ -1001,16 +1027,24 @@ def approximate_spectrum(self, E, Q, m_nu_squared=0): """ if self.use_final_states: - N_states = len(self.final_state_array[0]) - Q_states = Q+self.final_state_array[0]-np.max(self.final_state_array[0]) + N_states = len(self.final_state_array[0]) + Q_states = Q+self.final_state_array[0]-np.max(self.final_state_array[0]) + + #index = [np.where(((Q_states[i]-E)**2-m_nu_squared > 0) & (Q_states[i]-E > 0)) for i in range(N_states)] + #index = [np.where(E < Q_states[i] -mnu) for i in range(N_states)] + beta_rates_array = [self.beta_rates(E, Q_states[i], m_nu_squared)#, index[i]) + * self.final_state_array[1][i] + for i in range(N_states)] - #index = [np.where(((Q_states[i]-E)**2-m_nu_squared > 0) & (Q_states[i]-E > 0)) for i in range(N_states)] - #index = [np.where(E < Q_states[i] -mnu) for i in range(N_states)] - beta_rates_array = [self.beta_rates(E, Q_states[i], m_nu_squared)#, index[i]) - * self.final_state_array[1][i] - for i in range(N_states)] + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) - spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) + + # max_energy = self.max_final_state_energy_loss + # dE = self.energies[1]-self.energies[0]#E[1]-E[0] + # n_dE = round(max_energy/dE) + # e_lineshape = np.arange(-n_dE*dE, n_dE*dE, dE) + # spectrum_nomfs = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.beta_rates(E, Q, m_nu_squared) + # spectrum = convolve(spectrum_nomfs, self.final_states_interp(e_lineshape), mode='same') else: @@ -1085,6 +1119,9 @@ def f_i(self, K, gamma, mu, sigma): f = np.exp(gamma*(mu+K+gamma*sigma**2/2.))*erfc(z) return f/np.sum(f) + + + def more_accurate_simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): """ Still a simplified lineshape but helium is not just a gaussian @@ -1162,8 +1199,9 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1, h2_ """ This uses Gaussians of different mu and sigma for different gases """ + if self.plot_lineshape: - logger.info('Using gaussian multi gas scattering') + logger.info('Using simplified lineshape. Hydrogen proportion is {}'.format(self.hydrogen_proportion)) p0, p1, p2, p3 = self.lineshape_p[1], self.lineshape_p[3], self.lineshape_p[5], self.lineshape_p[7] q0, q1, q2, q3 = self.helium_lineshape_p[1], self.helium_lineshape_p[3], self.helium_lineshape_p[5], self.helium_lineshape_p[7] @@ -1171,9 +1209,9 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1, h2_ sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) - if sig0 < 6: - logger.warning('Scatter resolution < 6 eV. Setting to 6 eV') - sig0 = 6 + #if sig0 < 6: + # logger.warning('Scatter resolution {} < 6 eV. Setting to 6 eV'.format(sig0)) + # sig0 = 6 #shape = self.gauss_resolution_f(K, 1, sig0, Kcenter) #shape = np.zeros(len(K)) norm = 1. @@ -1183,6 +1221,10 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1, h2_ #plt.figure(figsize=(10,10)) + #scatter_peaks = np.array([[self.gauss_resolution_f(K, 1, p2[i]+p3[i]*sig0, -p0[i]+p1[i]*sig0+Kcenter)*self.mode_exp_scatter_peak_ratio(prob_b, prob_c, i+1), + # self.gauss_resolution_f(K, 1, q2[i]+q3[i]*sig0, -q0[i]+q1[i]*sig0+Kcenter)*self.mode_exp_scatter_peak_ratio(prob_b, prob_c, i+1)] for i in range(self.NScatters)]) + + for i in range(self.NScatters): # hydrogen scattering @@ -1224,6 +1266,43 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1, h2_ plt.plot(K, lineshape_rates/np.max(lineshape_rates), label='complex lineshape', color='purple', linestyle='--') return lineshape_rates""" + def scatter_peaks(self, K, Kcenter, FWHM): + + if self.plot_lineshape: + logger.info('Using simplified scatter peaks.') + + p0, p1, p2, p3 = self.lineshape_p[1], self.lineshape_p[3], self.lineshape_p[5], self.lineshape_p[7] + q0, q1, q2, q3 = self.helium_lineshape_p[1], self.helium_lineshape_p[3], self.helium_lineshape_p[5], self.helium_lineshape_p[7] + + + + sig0 = FWHM/float(2*np.sqrt(2*np.log(2))) + #if sig0 < 6: + # logger.warning('Scatter resolution {} < 6 eV. Setting to 6 eV'.format(sig0)) + # sig0 = 6 + #shape = self.gauss_resolution_f(K, 1, sig0, Kcenter) + #shape = np.zeros(len(K)) + norm = 1. + #hydrogen_scattering = np.array([self.gauss_resolution_f(K, 1, p2[i]+p3[i]*sig0, (-p0[i]+p1[i]*sig0)+Kcenter) for i in range(self.NScatters)]) + #helium_scattering = np.array([self.gauss_resolution_f(K, 1, q2[i]+q3[i]*sig0, (-q0[i]+q1[i]*sig0)+Kcenter) for i in range(self.NScatters)]) + + hydrogen_scattering = np.zeros((self.NScatters, len(K))) + helium_scattering = np.zeros((self.NScatters, len(K))) + + + for i in range(self.NScatters): + + # hydrogen scattering + mu = -p0[i]+p1[i]*sig0 + sig = p2[i]+p3[i]*sig0 + hydrogen_scattering[i] = self.gauss_resolution_f(K, 1, sig, mu+Kcenter) + + # helium scattering + mu_he = -q0[i]+q1[i]*sig0 + sig_he = q2[i]+q3[i]*sig0 + helium_scattering[i] = self.gauss_resolution_f(K, 1, sig_he, mu_he+Kcenter) + + return hydrogen_scattering, helium_scattering def running_mean(self, x, N): N_round = round(N) @@ -1231,6 +1310,7 @@ def running_mean(self, x, N): return (cumsum[int(N_round)::int(N_round)] - cumsum[:-int(N_round):int(N_round)]) / float(N_round) + def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., prob_b=None, prob_c=None, res=None, sig1=None, sig2=None, tilt=0., error=False): E = np.array(E) @@ -1323,6 +1403,7 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p #K_convolved = K_convolved[e_spec>=np.min(self.energies)] + if self.is_scattered: # scatter params if 'scatter_peak_ratio_p' not in self.model_parameter_names: @@ -1345,25 +1426,54 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p # simplified lineshape FWHM = 2.*np.sqrt(2.*np.log(2.))*res *self.width_scaling - # get lineshape + # add tail if not self.use_helium_scattering: tail, norm = self.simplified_ls(e_lineshape, 0, FWHM, prob_b, prob_c) + lineshape += tail + lineshape = lineshape/norm + K_convolved = convolve(spec, lineshape, mode='same') if self.plot_lineshape: logger.info('Using simplified lineshape model') + + elif self.use_frequency_dependent_lineshape: + Kbounds = [e_spec[0]] + list(self.scatter_factor_bounds) + [e_spec[-1]] + + peaks = [self.scatter_peaks(e_lineshape, 0, FWHM*self.width_factors[i]) for i in range(len(self.p_factors))] + #hydrogen_peaks, helium_peaks = self.scatter_peaks(e_lineshape, 0, FWHM) + scatter_peak_ratios = [self.mode_exp_scatter_peak_ratio(prob_b*self.p_factors[j], prob_c*self.q_factors[j], np.arange(self.NScatters)+1) for j in range(len(self.p_factors))] + norms = np.sum(scatter_peak_ratios, axis=1) + + lineshape = np.array([self.derived_two_gaussian_resolution(e_lineshape, res*self.width_factors[i], two_gaussian_mu_1, two_gaussian_mu_2) for i in range(len(self.p_factors))]) + hydrogen_tail = np.array([np.sum(np.multiply(peaks[i][0], scatter_peak_ratios[i][:, None]), axis=0)*h2_fraction for i in range(len(self.p_factors))]) + helium_tail = np.array([np.sum(peaks[i][1]*scatter_peak_ratios[i][:, None], axis=0)*(1-h2_fraction) for i in range(len(self.p_factors))]) + detector_response = (hydrogen_tail+helium_tail+lineshape)/norms[:,None] + + + + #tails_and_norms = [self.multi_gas_lineshape(e_lineshape, 0, FWHM, prob_b*self.p_factors[i], prob_c*self.q_factors[i], h2_fraction) for i in range(len(self.p_factors))] + K_convolved_segments = [convolve(spec, detector_response[i], mode='same')[np.logical_and(Kbounds[i]<=e_spec, e_spec<=Kbounds[i+1])] for i in range(len(self.p_factors))] + + K_convolved = np.concatenate(K_convolved_segments, axis=None) + + if self.plot_lineshape: + logger.info('Using two gas simplified lineshape model with frequency dependent p & q') + lineshape = detector_response[0] else: tail, norm = self.multi_gas_lineshape(e_lineshape, 0, FWHM, prob_b, prob_c, h2_fraction) + lineshape += tail + lineshape = lineshape/norm + K_convolved = convolve(spec, lineshape, mode='same') if self.plot_lineshape: logger.info('Using two gas simplified lineshape model') - lineshape += tail - lineshape = lineshape/norm + try: + K_convolved = np.interp(self.energies, e_spec, K_convolved) + except Exception as e: + print(np.shape(Kbounds)) + print(np.shape(self.p_factors)) + print(np.shape(e_spec), np.shape(K_convolved), np.shape(spec),np.shape(detector_response)) + raise e - # lineshape done now convolve - K_convolved = convolve(spec, lineshape, mode='same') - below_Kmin = np.where(e_spec < min(self.energies)) - #np.put(K_convolved, below_Kmin, np.zeros(len(below_Kmin))) - K_convolved = np.interp(self.energies, e_spec, K_convolved) - #K_convolved = K_convolved[e_spec>=np.min(self.energies)] #logger.info('Plotting lineshape now: {}'.format(self.plot_lineshape)) From aea654b1da26fc61e8ea5c5d47d641fbc0e6a86c Mon Sep 17 00:00:00 2001 From: cclaessens Date: Tue, 26 Apr 2022 22:46:29 +0200 Subject: [PATCH 160/199] moved iminuit setup to separate method --- .../processors/Fitters/BinnedDataFitter.py | 65 ++++++++++--------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index d500a691..97286072 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -86,6 +86,8 @@ def InternalRun(self): logger.info('Data is unbinned. Will histogram before fitting.') self.hist, _ = np.histogram(self.data[self.namedata], self.bins) + logger.info('Total counts: {}'.format(np.sum(self.hist))) + result_array, error_array = self.fit() # save results @@ -108,6 +110,27 @@ def model(self, x, A, mu, sigma): return f + def setup_minuit(self): + + self.m_binned = Minuit(self.negPoissonLogLikelihood, + self.initial_values, + name=self.parameter_names, + ) + + self.m_binned.errordef = 0.5 + self.m_binned.errors = self.parameter_errors + self.m_binned.throw_nan = True + self.m_binned.print_level = self.print_level + + for i, name in enumerate(self.parameter_names): + if name in self.fixes_dict.keys(): + self.m_binned.fixed[name]= self.fixes_dict[name] + #logger.info('Fixing {}'.format(name)) + else: + self.m_binned.fixed[name] = self.fixes[i] + self.m_binned.limits[name] = self.limits[i] + + def fit(self): # Now minimize neg log likelihood using iMinuit @@ -123,53 +146,37 @@ def fit(self): logger.info('Constraint means: {}'.format(self.constrained_means)) logger.info('Constraint widths: {}'.format(self.constrained_widths)) - m_binned = Minuit(self.negPoissonLogLikelihood, - self.initial_values, - name=self.parameter_names, - ) - - m_binned.errordef = 0.5 - m_binned.errors = self.parameter_errors - m_binned.throw_nan = True - m_binned.print_level = self.print_level - - for i, name in enumerate(self.parameter_names): - if name in self.fixes_dict.keys(): - m_binned.fixed[name]= self.fixes_dict[name] - #logger.info('Fixing {}'.format(name)) - else: - m_binned.fixed[name] = self.fixes[i] - m_binned.limits[name] = self.limits[i] + self.setup_minuit() # minimze - m_binned.simplex().migrad() + self.m_binned.simplex().migrad() #m_binned.hesse() #m_binned.minos() #self.param_states = m_binned.get_param_states() #logger.info(self.param_states) - self.m_binned = m_binned + # results - result_array = np.array(m_binned.values) - error_array = np.array(m_binned.errors) + result_array = np.array(self.m_binned.values) + error_array = np.array(self.m_binned.errors) if self.minos_intervals: self.minos_errors = {} for mcl in self.minos_cls: logger.info('Getting minos errors for CL = {}'.format(mcl)) try: - m_binned.minos(cl=mcl) + self.m_binned.minos(cl=mcl) except RuntimeError as e: - print(m_binned.params) + print(self.m_binned.params) raise e self.minos_errors[mcl] = {} if self.print_level: - logger.info(m_binned.merrors) - for k in m_binned.merrors.keys(): - self.minos_errors[mcl][k] = {'interval': [m_binned.merrors[k].lower, m_binned.merrors[k].upper], - 'number': m_binned.merrors[k].number, - 'name': m_binned.merrors[k].name, - 'is_valid': m_binned.merrors[k].is_valid} + logger.info(self.m_binned.merrors) + for k in self.m_binned.merrors.keys(): + self.minos_errors[mcl][k] = {'interval': [self.m_binned.merrors[k].lower, self.m_binned.merrors[k].upper], + 'number': self.m_binned.merrors[k].number, + 'name': self.m_binned.merrors[k].name, + 'is_valid': self.m_binned.merrors[k].is_valid} if self.print_level == 1: logger.info('Fit results: {}'.format(result_array)) From 29c34822ccd897b1db95dbe022b99cb58cb425ec Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 26 May 2022 22:09:45 -0500 Subject: [PATCH 161/199] p and q factors work for gaussian resolution in fake data generation. Freqeuncy variation can be propagated with MC sampling. reduce number of points in comlex lineshape array for fake data generation. --- mermithid/misc/FakeTritiumDataFunctions.py | 10 +- .../processors/Fitters/BinnedDataFitter.py | 18 +- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 17 +- .../TritiumSpectrum/FakeDataGenerator.py | 7 +- .../misc/MultiGasComplexLineShape.py | 228 +++++++++--------- 5 files changed, 150 insertions(+), 130 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 1d7b2331..df9999fe 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -313,24 +313,28 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape_rates.append(np.flipud(complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale_factors[i], ls_params[1], scatter_peak_ratio_p*p_factors[i], scatter_peak_ratio_q*q_factors[i], scatter_fraction, emitted_peak='dirac'))) elif resolution_function == 'gaussian_resolution' or resolution_function == 'gaussian': logger.warn("Scatter peak ratio function for lineshape with Gaussian resolution may not be up-to-date!") - lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, emitted_peak='dirac') + gaussian_widths = [ls_params[0]*f for f in ins_res_width_factors] + lineshape_rates = [np.flipud(complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(gaussian_widths[i], ls_params[1], scatter_peak_ratio_p*p_factors[i], scatter_peak_ratio_q*q_factors[i], scatter_fraction, emitted_peak='dirac')) for i in range(len(gaussian_widths))] else: logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) below_Kmin = np.where(K < Kmin) #Convolving - if (lineshape=='detailed_scattering' or lineshape=='detailed') and (resolution_function == 'simulated_resolution' or resolution_function == 'simulated'): + if (lineshape=='detailed_scattering' or lineshape=='detailed'):# and (resolution_function == 'simulated_resolution' or resolution_function == 'simulated'): convolved_segments = [] beta_rates = spectral_rate(K, Q, mnu, final_state_array) + plt.figure(figsize=(7,5)) for j in range(len(lineshape_rates)): #beta_rates = spectral_rate(K_segments[j], Q, mnu, final_state_array) + plt.plot(lineshape_rates[j]) convolved_j = convolve(beta_rates, lineshape_rates[j], mode='same') np.put(convolved_j, below_Kmin, np.zeros(len(below_Kmin))) #Only including the part of convolved_j that corresponds to the right values of K convolved_segments.append(convolved_j[np.logical_and(Kbounds[j]<=K, K<=Kbounds[j+1])]) #convolved.append(convolved_j) convolved = np.concatenate(convolved_segments, axis=None) + plt.savefig('varied_lineshapes.png', dpi=200) elif resolution_function=='gaussian': lineshape_rates = np.flipud(lineshape_rates) beta_rates = spectral_rate(K, Q, mnu, final_state_array) @@ -341,7 +345,7 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, beta_rates = spectral_rate(K, Q, mnu, final_state_array) convolved = convolve(beta_rates, lineshape_rates, mode='same') np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) - + return convolved diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index 97286072..fc8111ef 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -136,15 +136,15 @@ def fit(self): # Now minimize neg log likelihood using iMinuit if self.print_level > 0: logger.info('This is the plan:') - logger.info('Fitting data consisting of {} elements'.format(np.sum(self.hist))) - logger.info('Fit parameters: {}'.format(self.parameter_names)) - logger.info('Initial values: {}'.format(self.initial_values)) - logger.info('Initial error: {}'.format(self.parameter_errors)) - logger.info('Limits: {}'.format(self.limits)) - logger.info('Fixed in fit: {}'.format(self.fixes)) - logger.info('Constrained parameters: {}'.format([self.parameter_names[i] for i in self.constrained_parameters])) - logger.info('Constraint means: {}'.format(self.constrained_means)) - logger.info('Constraint widths: {}'.format(self.constrained_widths)) + logger.info('\tFitting data consisting of {} elements'.format(np.sum(self.hist))) + logger.info('\tFit parameters: {}'.format(self.parameter_names)) + logger.info('\tInitial values: {}'.format(self.initial_values)) + logger.info('\tInitial error: {}'.format(self.parameter_errors)) + logger.info('\tLimits: {}'.format(self.limits)) + logger.info('\tFixed in fit: {}'.format(self.fixes)) + logger.info('\tConstrained parameters: {}'.format([self.parameter_names[i] for i in self.constrained_parameters])) + logger.info('\tConstraint means: {}'.format(self.constrained_means)) + logger.info('\tConstraint widths: {}'.format(self.constrained_widths)) self.setup_minuit() diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 4b178ac6..ff42d196 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -97,7 +97,7 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, T = BinnedTritiumMLFitter("TritiumFitter") T.InternalConfigure(fit_config_dict) - T.print_level=1 + #T.print_level=1 #T.error_scaling = error_scaling T.integrate_bins = True logger.info('Energy stepsize: {}'.format(T.denergy)) @@ -360,6 +360,10 @@ def InternalConfigure(self, config_dict): self.scatter_factor_bounds = self.Energy(self.ins_res_bounds_freq) self.p_factors = np.array(reader.read_param(config_dict, 'p_factors', [1])) self.q_factors = np.array(reader.read_param(config_dict, 'q_factors', [1])) + self.p_factors_mean = np.array(reader.read_param(config_dict, 'p_factors_mean', [1])) + self.q_factors_mean = np.array(reader.read_param(config_dict, 'q_factors_mean', [1])) + self.p_factors_width = np.array(reader.read_param(config_dict, 'p_factors_width', [1])) + self.q_factors_width = np.array(reader.read_param(config_dict, 'q_factors_width', [1])) self.width_factors = np.array(reader.read_param(config_dict, 'res_width_factors', [1])) #self.scatter_factor_bounds = np.sort(self.Energy([25.90*10**9, 25.91*10**9, 25.92*10**9, 25.925*10**9, 25.93*10**9, 25.94*10**9, 25.96*10**9, 25.965*10**9, 25.9675*10**9, 25.968*10**9, 25.97*10**9, 25.98*10**9])) @@ -680,7 +684,10 @@ def SampleConvertAndFit(self, sampled_parameters={}, params= []): # ========================================================================= def Gaussian_sample(self, mean, width): np.random.seed() - return np.random.randn()*width+mean + if isinstance(width, list): + return np.random.randn(len(width))*width+mean + else: + return np.random.randn()*width+mean def Beta_sample(self, mean, width): np.random.seed() @@ -769,6 +776,12 @@ def SamplePriors(self, sampled_parameters): self.parameter_samples['endpoint'] = self.endpoint self.fix_endpoint = True sample_values.append(self.endpoint) + if 'frequency_dependent_response' in sampled_parameters.keys() and sampled_parameters['frequency_dependent_response']: + self.p_factors = self.Gaussian_sample(self.p_factors_mean, self.p_factors_width) + self.q_factors = self.Gaussian_sample(self.q_factors_mean, self.q_factors_width) + sample_values.extend([np.mean(self.p_factors), np.mean(self.q_factors)]) + self.use_frequency_dependent_lineshape = True + logger.info('Samples are: {}'.format(sample_values)) #logger.info('Fit parameters: \n{}\nFixed: {}'.format(self.parameter_names, self.fixes)) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index c5fb0bf9..69a5dbec 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -230,7 +230,7 @@ def InternalConfigure(self, params): 'sigma_array': self.sigma_array, # This is an important parameter which determines how finely resolved the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to be increased for larger datasets. - 'num_points_in_std_array':35846, + 'num_points_in_std_array':4096, #35846, 'base_shape': 'dirac', 'path_to_osc_strengths_files': self.path_to_osc_strengths_files, 'path_to_scatter_spectra_file':self.path_to_scatter_spectra_file, @@ -246,6 +246,8 @@ def InternalConfigure(self, params): self.complexLineShape.Configure(complexLineShape_config) logger.info('Checking existence of scatter spectra files') self.complexLineShape.check_existence_of_scatter_file() + lineshape_array = self.complexLineShape.std_eV_array() + self.lineshape_stepize = lineshape_array[1]-lineshape_array[0] else: logger.error("'detailed_or_simplified' is neither 'detailed' nor 'simplified'") return False @@ -373,7 +375,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 logger.info('Stepsize is {} eV'.format(step_size)) #Options of kinetic energies to be sampled - self.Koptions = np.arange(Kmin_eff, Kmax_eff, step_size) + self.Koptions = np.arange(Kmin_eff, Kmax_eff, self.lineshape_stepize) if efficiency_dict is not None: logger.info('Evaluating efficiencies') @@ -460,6 +462,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 #if self.ins_res_width_bounds==None: temp_Koptions, temp_probsS, temp_probsB = self.Koptions, probsS, probsB split_Koptions, split_probsS, split_probsB = [], [], [] + print(len(temp_Koptions), len(temp_probsS)) for i in range(len(self.channel_bounds)): split_Koptions.append(temp_Koptions[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) split_probsS.append(temp_probsS[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 062ea131..5fdacd22 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -51,7 +51,7 @@ def InternalConfigure(self, params): # Read other parameters self.bins_choice = reader.read_param(params, 'bins_choice', []) self.gases = reader.read_param(params, 'gases', ["H2", "Kr", "He", "Ar"]) - # when self.fix_gas_composition and self.fix_width_scale_factor are both True, + # when self.fix_gas_composition and self.fix_width_scale_factor are both True, # fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor is used, # Then the first N-1 gas compositions are set below through self.scatter_fractions_for_gases # Otherwise, fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio is used @@ -117,7 +117,7 @@ def InternalConfigure(self, params): raise IOError('Path to osc strengths files does not exist') # Read shake parameters from JSON file if self.base_shape == 'shake': - self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) + self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) return True def InternalRun(self): @@ -190,7 +190,7 @@ def std_lorenztian_17keV(self): x_array = self.std_eV_array() ans = lorentzian(x_array,0,kr_line_width) return ans - + #A Dirac delta functin def std_dirac(self): x_array = self.std_eV_array() @@ -204,7 +204,7 @@ def std_dirac(self): logger.warning('Lineshape will shift spectrum by > 1 eV') raise ValueError('problem with std_eV_array()') return ans - + # A gaussian function def gaussian(self, x_array, A, sigma, mu): f = A*(1./(sigma*np.sqrt(2*np.pi)))*np.exp(-(((x_array-mu)/sigma)**2.)/2.) @@ -278,7 +278,7 @@ def composite_gaussian_lorentzian(self, sigma): x_array = self.std_eV_array() w_g = x_array/sigma gamma = self.ratio_gamma_to_sigma*sigma - w_l = x_array/gamma + w_l = x_array/gamma lorentzian = 1./(gamma*np.pi)*1./(1+(w_l**2)) gaussian = 1./(np.sqrt(2.*np.pi)*sigma)*np.exp(-0.5*w_g**2) p = self.gaussian_proportion @@ -289,7 +289,7 @@ def elevated_gaussian(self, elevation_factor, sigma): x_array = self.std_eV_array() w_g = x_array/sigma gamma = self.ratio_gamma_to_sigma*sigma - w_l = x_array/gamma + w_l = x_array/gamma lorentzian = 1./(gamma*np.pi)*1./(1+(w_l**2)) gaussian = 1./(np.sqrt(2.*np.pi)*sigma)*np.exp(-0.5*w_g**2) modified_guassian_function = gaussian*(1 + elevation_factor*lorentzian) @@ -333,7 +333,7 @@ def another_scatter(self, input_spectrum, gas_type): f = signal.convolve(single,input_spectrum,mode='same') f_normed = self.normalize(f) return f_normed - + def radiation_loss_f(self): radiation_loss_data_file_path = self.path_to_missing_track_radiation_loss_data_numpy_file + '/missing_track_radiation_loss.npy' data_for_missing_track_radiation_loss = np.load(radiation_loss_data_file_path, allow_pickle = True) @@ -349,7 +349,7 @@ def radiation_loss_f(self): return f_radiation_energy_loss # Convolves the scatter functions and saves - # the results to a .npy file. + # the results to a .npy file. def generate_scatter_convolution_file(self): t = time.time() scatter_spectra_single_gas = {} @@ -387,7 +387,7 @@ def generate_scatter_convolution_file(self): mark_first_nonzero_component = 1 else: scatter_to_add = scatter_spectra_single_gas[gas_type][str(component).zfill(2)] - current_full_scatter = self.normalize(signal.convolve(current_full_scatter, scatter_to_add, mode='same')) + current_full_scatter = self.normalize(signal.convolve(current_full_scatter, scatter_to_add, mode='same')) scatter_spectra[entry_str] = current_full_scatter np.save(os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy'), scatter_spectra) elapsed = time.time() - t @@ -399,18 +399,18 @@ def generate_scatter_convolution_file(self): # If not, this function calls generate_scatter_convolution_file. # This function also checks to make sure that the scatter file have the correct # number of entries and correct number of points in the SELA, and if not, it generates a fresh file. - # When the variable regenerate is set as True, it generates a fresh file + # When the variable regenerate is set as True, it generates a fresh file def check_existence_of_scatter_file(self, regenerate = True): gases = self.gases if regenerate == True: logger.info('generate fresh scatter file') self.generate_scatter_convolution_file() - else: + else: directory = os.listdir(self.path_to_scatter_spectra_file) strippeddirs = [s.strip('\n') for s in directory] if 'scatter_spectra.npy' not in strippeddirs: self.generate_scatter_convolution_file() - test_file = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + test_file = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') test_dict = np.load(test_file, allow_pickle = True) N = len(self.gases) if len(test_dict.item()) != sum([comb(M + N -1, N -1) for M in range(1, self.max_scatters+1)]): @@ -419,7 +419,7 @@ def check_existence_of_scatter_file(self, regenerate = True): test_dict = np.load(test_file, allow_pickle = True) gas_str = gases[0] + '01' for gas in self.gases[1:]: - gas_str += gas + '00' + gas_str += gas + '00' if gas_str not in list(test_dict.item().keys()): print('Gas species not matching, generating fresh files') self.generate_scatter_convolution_files() @@ -432,7 +432,7 @@ def convolve_gaussian(self, func_to_convolve, gauss_FWHM_eV): ans = signal.convolve(resolution_f, func_to_convolve,mode='same') ans_normed = self.normalize(ans) return ans_normed - + def convolve_composite_gaussian(self, func_to_convolve, A_array, sigma_array): resolution_f = self.composite_gaussian(A_array, sigma_array) ans = signal.convolve(resolution_f, func_to_convolve, mode='same') @@ -494,7 +494,7 @@ def convolve_ins_resolution(self, working_spectrum): convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum - + def combine_four_trap_resolution_from_txt(self, trap_weights): if self.sample_ins_resolution_errors: weight_array = np.random.normal(trap_weights['weights'], trap_weights['errors']) @@ -509,7 +509,7 @@ def combine_four_trap_resolution_from_txt(self, trap_weights): y_data_combined = weight_array[0]*y_data_array[0] + weight_array[1]*y_data_array[1] + weight_array[2]*y_data_array[2] + weight_array[3]*y_data_array[3] y_err_data_combined = np.sqrt((weight_array[0]*y_err_data_array[0])**2 + (weight_array[1]*y_err_data_array[1])**2 + (weight_array[2]*y_err_data_array[2])**2 + (weight_array[3]*y_err_data_array[3])**2) return x_data, y_data_combined, y_err_data_combined - + def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_array): x_data, y_data_combined, y_err_data_combined = self.combine_four_trap_resolution_from_txt(weight_array) if self.sample_ins_resolution_errors: @@ -654,8 +654,8 @@ def chi_2_Poisson_simulated_resolution_scaled_fit_recon_eff(self, bin_centers, d def reduced_chi2_Pearson_Neyman_composite(self, data_hist_freq, fit_Hz, number_of_parameters): nonzero_bins_index = np.where(data_hist_freq != 0)[0] zero_bins_index = np.where(data_hist_freq == 0)[0] - fit_Hz_nonzero = fit_Hz[nonzero_bins_index] - data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] fit_Hz_zero = fit_Hz[zero_bins_index] data_Hz_zero = data_hist_freq[zero_bins_index] chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) @@ -722,7 +722,7 @@ def spectrum_func(self, bins_Hz, *p0): prob_parameter = p0[3] N = len(self.gases) scatter_proportion = p0[4:3+N] - + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) en_loss_array = self.std_eV_array() en_loss_array_min = en_loss_array[0] @@ -772,7 +772,7 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): scatter_proportion_max = 1 N = len(self.gases) p0_guess = [B_field_guess, FWHM_guess, amplitude_guess, prob_parameter_guess] + [scatter_proportion_guess]*(N-1) - p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min] + [scatter_proportion_min]*(N-1), + p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min] + [scatter_proportion_min]*(N-1), [B_field_max, FWHM_eV_max, amplitude_max, prob_parameter_max] + [scatter_proportion_max]*(N-1) ) # Actually do the fitting params , cov = curve_fit(self.spectrum_func, bins_Hz_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) @@ -793,16 +793,16 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): prob_parameter_fit_err = perr[3] scatter_proportion_fit_err = list(perr[4:3+N])+[np.sqrt(sum(perr[4:3+N]**2))] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func(bins_Hz, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) - + nonzero_bins_index = np.where(data_hist_freq != 0)[0] zero_bins_index = np.where(data_hist_freq == 0)[0] - fit_Hz_nonzero = fit_Hz[nonzero_bins_index] - data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] fit_Hz_zero = fit_Hz[zero_bins_index] data_Hz_zero = data_hist_freq[zero_bins_index] chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) @@ -891,7 +891,7 @@ def spectrum_func_1(self, bins_Hz, *p0): FWHM_G_eV = p0[1] amplitude = p0[2] prob_parameter = p0[3] - + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) en_loss_array = self.std_eV_array() en_loss_array_min = en_loss_array[0] @@ -929,9 +929,9 @@ def fit_data_1(self, freq_bins, data_hist_freq): amplitude_max = np.sum(data_hist_freq)*3 prob_parameter_min = 1e-5 prob_parameter_max = 1 - - # p0_guess = [B_field_guess, FWHM_guess, amplitude_guess, prob_parameter_guess] - # p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min], + + # p0_guess = [B_field_guess, FWHM_guess, amplitude_guess, prob_parameter_guess] + # p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min], # [B_field_max, FWHM_eV_max, amplitude_max, prob_parameter_max]) # Actually do the fitting # params , cov = curve_fit(self.spectrum_func_1, bins_Hz_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) @@ -959,7 +959,7 @@ def fit_data_1(self, freq_bins, data_hist_freq): amplitude_fit_err = perr[2] prob_parameter_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_1(bins_Hz, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 @@ -967,8 +967,8 @@ def fit_data_1(self, freq_bins, data_hist_freq): nonzero_bins_index = np.where(data_hist_freq != 0)[0] zero_bins_index = np.where(data_hist_freq == 0)[0] - fit_Hz_nonzero = fit_Hz[nonzero_bins_index] - data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] fit_Hz_zero = fit_Hz[zero_bins_index] data_Hz_zero = data_hist_freq[zero_bins_index] reduced_chi2 = self.reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = 4) @@ -1088,7 +1088,7 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): amplitude_max = np.sum(data_hist_freq)*3 prob_parameter_min = 1e-5 prob_parameter_max = 1 - + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess] p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max)] # Actually do the fitting @@ -1111,7 +1111,7 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): amplitude_fit_err = perr[1] survival_prob_fit_err = perr[2] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_ftc(bins_Hz, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 @@ -1175,7 +1175,7 @@ def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak=' current_working_spectrum = self.convolve_ins_resolution_combining_four_trap(current_working_spectrum, self.trap_weights) else: current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) - + zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum N = len(self.gases) @@ -1265,7 +1265,7 @@ def fit_data_ftc_2(self, freq_bins, data_hist_freq): prob_parameter_fit_err = perr[2] scatter_proportion_fit_err = list(perr[3:2+N])+[np.sqrt(sum(perr[3:2+N]**2))] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_ftc_2(bins_Hz, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 @@ -1343,7 +1343,7 @@ def make_spectrum_smeared_triangle(self, prob_parameter, center, scale1, scale2, return current_full_spectrum def spectrum_func_smeared_triangle(self, bins_Hz, *p0): - + B_field = p0[0] amplitude = p0[1] prob_parameter = p0[2] @@ -1352,7 +1352,7 @@ def spectrum_func_smeared_triangle(self, bins_Hz, *p0): scale2 = p0[5] exponent = p0[6] sigma = p0[7] - + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) en_loss_array = self.std_eV_array() en_loss_array_min = en_loss_array[0] @@ -1364,7 +1364,7 @@ def spectrum_func_smeared_triangle(self, bins_Hz, *p0): x_eV_minus_line = Constants.kr_line()*1000 - x_eV zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - + full_spectrum = self.make_spectrum_smeared_triangle(prob_parameter, center, scale1, scale2, exponent, sigma) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) @@ -1397,9 +1397,9 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print width_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) - ConversionFunctions.Energy(bins_Hz[-1], B_field_guess) exponent_min = 0.5 exponent_max = 2 - + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess, center_guess, scale_guess, scale_guess, exponent_guess, sigma_guess] - p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (center_min, center_max), (width_min, width_max), (width_min, width_max), (exponent_min, exponent_max), (width_min, width_max)] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (center_min, center_max), (width_min, width_max), (width_min, width_max), (exponent_min, exponent_max), (width_min, width_max)] #step_size = [1e-6, 5, 500, 0.1] # Actually do the fitting print(p0_guess) @@ -1432,12 +1432,12 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print exponent_fit_err = perr[6] sigma_fit_err = perr[7] total_counts_fit_err = amplitude_fit_err - + fit_Hz = spectrum_func(bins_Hz,*params) fit_keV = flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = flip_array(bins_keV) - reduced_chi2 = reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = len(params)) + reduced_chi2 = reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = len(params)) if print_params == True: output_string = 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) output_string += '-----------------\n' @@ -1486,7 +1486,7 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results - + #fitting with commposite gaussian lorentzian resolution function and fixed scatter fraction def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion(self, survival_prob, sigma, emitted_peak='shake'): p = self.scatter_proportion @@ -1525,10 +1525,10 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion(self, s return current_full_spectrum def spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] - survival_prob = p0[2] + survival_prob = p0[2] sigma = p0[3] x_eV = ConversionFunctions.Energy(bins_Hz, B_field) @@ -1554,7 +1554,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion(self, freq_b self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -1606,13 +1606,13 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion(self, freq_b survival_prob_fit_err = perr[2] sigma_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -1686,9 +1686,9 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion_and_sur return current_full_spectrum def spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] - amplitude = p0[1] + amplitude = p0[1] sigma = p0[2] x_eV = ConversionFunctions.Energy(bins_Hz, B_field) @@ -1714,7 +1714,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -1764,13 +1764,13 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival amplitude_fit_err = perr[1] sigma_fit_err = perr[2] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -1842,9 +1842,9 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability(self, return current_full_spectrum def spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] - amplitude = p0[1] + amplitude = p0[1] sigma = p0[2] N = len(self.gases) scatter_proportion = p0[3: 2+N] @@ -1872,7 +1872,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability(self, freq self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -1924,13 +1924,13 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability(self, freq sigma_fit_err = perr[2] scatter_proportion_fit_err = list(perr[3:2+N]) + [np.sqrt(sum(perr[3:2+N]**2))] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2004,9 +2004,9 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability_parti return current_full_spectrum def spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] - amplitude = p0[1] + amplitude = p0[1] sigma = p0[2] N = len(self.free_gases) scatter_proportion = p0[3: 2+N] @@ -2034,7 +2034,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_ self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -2086,13 +2086,13 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_ sigma_fit_err = perr[2] scatter_proportion_fit_err = list(perr[3:2+N]) + [np.sqrt(sum(perr[3:2+N]**2))] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2174,10 +2174,10 @@ def make_spectrum_elevated_gaussian_fixed_scatter_proportion(self, survival_prob return current_full_spectrum def spectrum_func_elevated_gaussian_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] - survival_prob = p0[2] + survival_prob = p0[2] sigma = p0[3] elevation_factor = p0[4] @@ -2204,7 +2204,7 @@ def fit_data_elevated_gaussian_fixed_scatter_proportion(self, freq_bins, data_hi self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -2261,13 +2261,13 @@ def fit_data_elevated_gaussian_fixed_scatter_proportion(self, freq_bins, data_hi sigma_fit_err = perr[3] elevation_factor_fit_err = perr[4] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_elevated_gaussian_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2342,7 +2342,7 @@ def make_spectrum_composite_gaussian_fixed_scatter_proportion(self, survival_pro return current_full_spectrum def spectrum_func_composite_gaussian_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] survival_prob = p0[2] @@ -2370,7 +2370,7 @@ def fit_data_composite_gaussian_fixed_scatter_proportion(self, freq_bins, data_h self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -2423,13 +2423,13 @@ def fit_data_composite_gaussian_fixed_scatter_proportion(self, freq_bins, data_h amplitude_fit_err = perr[1] survival_prob_fit_err = perr[2] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2498,7 +2498,7 @@ def make_spectrum_composite_gaussian_pedestal_factor_fixed_scatter_proportion(se return current_full_spectrum def spectrum_func_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] survival_prob = p0[2] @@ -2527,7 +2527,7 @@ def fit_data_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, f self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -2582,13 +2582,13 @@ def fit_data_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, f survival_prob_fit_err = perr[2] pedestal_factor_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_pedestal_factor_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2658,7 +2658,7 @@ def make_spectrum_composite_gaussian_scaled_fixed_scatter_proportion(self, survi return current_full_spectrum def spectrum_func_composite_gaussian_scaled_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] survival_prob = p0[2] @@ -2687,7 +2687,7 @@ def fit_data_composite_gaussian_scaled_fixed_scatter_proportion(self, freq_bins, self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -2742,13 +2742,13 @@ def fit_data_composite_gaussian_scaled_fixed_scatter_proportion(self, freq_bins, survival_prob_fit_err = perr[2] scale_factor_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_scaled_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2819,7 +2819,7 @@ def make_spectrum_simulated_resolution_scaled_fixed_scatter_proportion(self, sur return current_full_spectrum def spectrum_func_simulated_resolution_scaled_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] survival_prob = p0[2] @@ -2848,7 +2848,7 @@ def fit_data_simulated_resolution_scaled_fixed_scatter_proportion(self, freq_bin self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -2903,13 +2903,13 @@ def fit_data_simulated_resolution_scaled_fixed_scatter_proportion(self, freq_bin survival_prob_fit_err = perr[2] scale_factor_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2976,7 +2976,7 @@ def make_spectrum_simulated_resolution_scaled_fit_recon_eff(self, survival_prob, return current_full_spectrum def spectrum_func_simulated_resolution_scaled_fit_recon_eff(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] survival_prob = p0[2] @@ -3008,7 +3008,7 @@ def fit_data_simulated_resolution_scaled_fit_recon_eff(self, freq_bins, data_his self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -3071,13 +3071,13 @@ def fit_data_simulated_resolution_scaled_fit_recon_eff(self, freq_bins, data_his recon_eff_b_fit_err = perr[5] recon_eff_c_fit_err = perr[6] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_recon_eff(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -3151,7 +3151,7 @@ def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale return current_full_spectrum def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] scale_factor = p0[2] @@ -3192,8 +3192,8 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, t = time.time() self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN - bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - if self.use_quad_trap_eff_interp == True: + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + if self.use_quad_trap_eff_interp == True: quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -3221,7 +3221,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, survival_probability_min = 1e-5 survival_probability_max = 1 scatter_fraction_min = 1e-5 - scatter_fraction_max = 1 + scatter_fraction_max = 1 scale_factor_min = 1e-5 scale_factor_max = 5 scatter_peak_ratio_parameter_min = 1e-5 @@ -3253,7 +3253,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, scatter_peak_ratio_q_fit = params[5] total_counts_fit = amplitude_fit logger.info('\n'+str(m_binned.params)) - scatter_fraction_fit = params[6:5+N]+[1- sum(params[6:5+N])] + scatter_fraction_fit = params[6:5+N]+[1- sum(params[6:5+N])] perr = m_binned.errors[0:] B_field_fit_err = perr[0] @@ -3264,14 +3264,14 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, scatter_peak_ratio_q_fit_err = perr[5] total_counts_fit_err = amplitude_fit_err scatter_fraction_fit_err = perr[6:5+N]+[np.sqrt(sum(np.array(perr[6:5+N])**2))] - + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) correlation_matrix = m_binned.covariance.correlation() - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -3318,7 +3318,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, return dictionary_of_fit_results - def make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(self, gauss_FWHM_eV, survival_probability, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='shake'): + def make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(self, gauss_FWHM_eV, survival_probability, scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, emitted_peak='shake'): p = np.zeros(len(self.gases)) p[0:-1] = scatter_fraction p[-1] = 1 - sum(scatter_fraction) @@ -3337,7 +3337,7 @@ def make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(self, gauss_FWHM_eV current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_b*M**scatter_peak_ratio_c) + scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_p*M**( -self.factor*scatter_peak_ratio_p + scatter_peak_ratio_q))#np.exp(-1.*scatter_peak_ratio_b*M**scatter_peak_ratio_c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) @@ -3354,7 +3354,7 @@ def make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(self, gauss_FWHM_eV return current_full_spectrum def spectrum_func_gaussian_resolution_fit_scatter_peak_ratio(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] gauss_FWHM_eV = p0[2] @@ -3396,7 +3396,7 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - if self.use_quad_trap_eff_interp == True: + if self.use_quad_trap_eff_interp == True: quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -3420,7 +3420,7 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi survival_probability_min = 1e-5 survival_probability_max = 1 scatter_fraction_min = 1e-5 - scatter_fraction_max = 1 + scatter_fraction_max = 1 scale_factor_min = 1e-5 scale_factor_max = 5 scatter_peak_ratio_parameter_min = 1e-5 @@ -3452,7 +3452,7 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi scatter_peak_ratio_c_fit = params[5] total_counts_fit = amplitude_fit logger.info('\n'+str(m_binned.params)) - scatter_fraction_fit = params[6:5+N]+[1- sum(params[6:5+N])] + scatter_fraction_fit = params[6:5+N]+[1- sum(params[6:5+N])] perr = m_binned.errors[0:] B_field_fit_err = perr[0] @@ -3463,13 +3463,13 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi scatter_peak_ratio_c_fit_err = perr[5] total_counts_fit_err = amplitude_fit_err scatter_fraction_fit_err = perr[6:5+N]+[np.sqrt(sum(np.array(perr[6:5+N])**2))] - + fit_Hz = self.spectrum_func_gaussian_resolution_fit_scatter_peak_ratio(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -3512,11 +3512,11 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi 'data_hist_freq': data_hist_freq, 'reduced_chi2': reduced_chi2 } - + return dictionary_of_fit_results def generate_scatter_peaks(self): - + p = np.zeros(len(self.gases)) scatter_fraction = self.scatter_fractions_for_gases p[0:-1] = scatter_fraction @@ -3534,7 +3534,7 @@ def generate_scatter_peaks(self): current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() elif emitted_peak == 'dirac': current_working_spectrum = self.std_dirac() - + scale_factor = 1 current_working_spectrum = self.convolve_simulated_resolution_scaled(current_working_spectrum, scale_factor) zeroth_order_peak = current_working_spectrum @@ -3570,7 +3570,7 @@ def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_ return current_full_spectrum def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(self, bins_Hz, eff_array, scatter_peaks, *p0): - + B_field = p0[0] amplitude = p0[1] survival_probability = p0[2] @@ -3608,8 +3608,8 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c t = time.time() self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN - bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - if self.use_quad_trap_eff_interp == True: + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + if self.use_quad_trap_eff_interp == True: quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -3636,7 +3636,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c survival_probability_min = 1e-5 survival_probability_max = 1 scatter_fraction_min = 1e-5 - scatter_fraction_max = 1 + scatter_fraction_max = 1 scale_factor_min = 1e-5 scale_factor_max = 5 scatter_peak_ratio_parameter_min = 1e-5 @@ -3648,7 +3648,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c p0_guess = [B_field_guess, amplitude_guess, survival_probability_guess, scatter_peak_ratio_parameter_guess, scatter_peak_ratio_parameter_guess] p0_bounds = [(B_field_min,B_field_max), (amplitude_min, amplitude_max), (survival_probability_min, survival_probability_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max)] parameter_names = ['B field','amplitude', 'survival probability','scatter peak ratio param p', 'scatter peak ratio param q'] - + scatter_peaks = self.generate_scatter_peaks() # Actually do the fitting m_binned = Minuit(lambda p: self.chi_2_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(bins_Hz, data_hist_freq, eff_array, scatter_peaks, p), p0_guess, name = parameter_names) @@ -3668,7 +3668,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c scatter_peak_ratio_p_fit = params[3] scatter_peak_ratio_q_fit = params[4] total_counts_fit = amplitude_fit - logger.info('\n'+str(m_binned.params)) + logger.info('\n'+str(m_binned.params)) perr = m_binned.errors[0:] B_field_fit_err = perr[0] @@ -3677,14 +3677,14 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c scatter_peak_ratio_p_fit_err = perr[3] scatter_peak_ratio_q_fit_err = perr[4] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(bins_Hz, eff_array, scatter_peaks, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) correlation_matrix = m_binned.covariance.correlation() - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) From 410cc3209d5b0403eb9e4c95e2283f4bc0049066 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Fri, 27 May 2022 22:09:08 -0500 Subject: [PATCH 162/199] added correlated parameters in frequentist analysis --- .../processors/Fitters/BinnedDataFitter.py | 21 ++++- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 91 ++++++++----------- .../TritiumSpectrum/FakeDataGenerator.py | 2 +- 3 files changed, 56 insertions(+), 58 deletions(-) diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index fc8111ef..f1c4b999 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -62,8 +62,10 @@ def InternalConfigure(self, params): self.binned_data = reader.read_param(params, 'binned_data', False) self.print_level = reader.read_param(params, 'print_level', 1) self.constrained_parameters = reader.read_param(params, 'constrained_parameter_indices', []) - self.constrained_means = reader.read_param(params, 'constrained_parameter_means', []) - self.constrained_widths = reader.read_param(params, 'constrained_parameter_widths', []) + self.constrained_means = np.array(reader.read_param(params, 'constrained_parameter_means', [])) + self.constrained_widths = np.array(reader.read_param(params, 'constrained_parameter_widths', [])) + self.correlated_parameters = reader.read_param(params, 'correlated_parameter_indices', []) + self.cov_matrix = np.array(reader.read_param(params, 'covariance_matrix', [])) self.minos_cls = reader.read_param(params, 'minos_confidence_level_list', []) self.minos_intervals = reader.read_param(params,'find_minos_intervals', False) @@ -145,6 +147,8 @@ def fit(self): logger.info('\tConstrained parameters: {}'.format([self.parameter_names[i] for i in self.constrained_parameters])) logger.info('\tConstraint means: {}'.format(self.constrained_means)) logger.info('\tConstraint widths: {}'.format(self.constrained_widths)) + logger.info('\tCorrelated parameters: {}'.format([self.parameter_names[i] for i in self.correlated_parameters])) + self.setup_minuit() @@ -220,6 +224,17 @@ def negPoissonLogLikelihood(self, params): # constrained parameters if len(self.constrained_parameters) > 0: for i, param in enumerate(self.constrained_parameters): - neg_ll += 0.5 * ((params[param] - self.constrained_means[i])/ self.constrained_widths[i])**2 + # only do uncorrelated parameters + if not param in self.correlated_parameters: + neg_ll += 0.5 * ((params[param] - self.constrained_means[i])/ self.constrained_widths[i])**2 + 0.5*np.log(2*np.pi) + np.log(self.constrained_widths[i]) + + if len(self.correlated_parameters) > 0: + dim = len(self.correlated_parameters) + constrained_indices = np.in1d(self.constrained_parameters, self.correlated_parameters).nonzero()[0] + param_indices = self.correlated_parameters + neg_ll += 0.5*(np.log(np.linalg.det(self.cov_matrix)) + \ + np.dot(np.transpose(np.subtract(params[param_indices], self.constrained_means[constrained_indices])), \ + np.dot(np.linalg.inv(self.cov_matrix), np.subtract(params[param_indices], self.constrained_means[constrained_indices]))) + \ + dim*np.log(2*np.pi)) return neg_ll diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index ff42d196..9598cb7a 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -21,6 +21,7 @@ import numpy as np from scipy import constants from scipy.special import erfc +from scipy.signal import convolve import matplotlib.pyplot as plt from scipy.interpolate import interp1d @@ -30,7 +31,7 @@ logger = morphologging.getLogger(__name__) from mermithid.processors.Fitters import BinnedDataFitter -from mermithid.misc.FakeTritiumDataFunctions import * +from mermithid.misc.FakeTritiumDataFunctions import fermi_func, pe, Ee, GF, Vud, Mnuc2 from mermithid.processors.misc.KrComplexLineShape import KrComplexLineShape from mermithid.misc.DetectionEfficiencyUtilities import pseudo_integrated_efficiency, integrated_efficiency, power_efficiency @@ -176,6 +177,7 @@ def InternalConfigure(self, config_dict): self.use_helium_scattering = reader.read_param(config_dict, 'use_helium_scattering', False) self.use_frequency_dependent_lineshape = reader.read_param(config_dict, 'use_frequency_dependent_lineshape', True) self.correlated_p_q_scale = reader.read_param(config_dict, 'correlated_p_q_scale', False) + self.correlated_p_q = reader.read_param(config_dict, 'correlated_p_q', False) self.derived_two_gaussian_model = reader.read_param(config_dict, 'derived_two_gaussian_model', True) @@ -288,15 +290,18 @@ def InternalConfigure(self, config_dict): self.scatter_peak_ratio_q = self.scatter_peak_ratio_q_mean #Adding correlated parameters (will later incorporate into morpho processor) - self.b_c_corr = reader.read_param(config_dict, 'b_c_corr', 0) - self.b_scale_corr = reader.read_param(config_dict, 'b_scale_corr', 0) - self.c_scale_corr = reader.read_param(config_dict, 'c_scale_corr', 0) + self.p_q_corr = reader.read_param(config_dict, 'p_q_corr', 0) + self.p_scale_corr = reader.read_param(config_dict, 'p_scale_corr', 0) + self.q_scale_corr = reader.read_param(config_dict, 'q_scale_corr', 0) stds = [self.scatter_peak_ratio_p_width, self.scatter_peak_ratio_q_width, self.scale_width] - self.b_c_scale_cov_matrix = [[stds[0]**2, self.b_c_corr*stds[0]*stds[1], self.b_scale_corr*stds[0]*stds[2]], - [self.b_c_corr*stds[1]*stds[0], stds[1]**2, self.c_scale_corr*stds[1]*stds[2]], - [self.b_scale_corr*stds[2]*stds[0], self.c_scale_corr*stds[2]*stds[1], stds[2]**2]] + self.p_q_scale_cov_matrix = [[stds[0]**2, self.p_q_corr*stds[0]*stds[1], self.p_scale_corr*stds[0]*stds[2]], + [self.p_q_corr*stds[1]*stds[0], stds[1]**2, self.q_scale_corr*stds[1]*stds[2]], + [self.p_scale_corr*stds[2]*stds[0], self.q_scale_corr*stds[2]*stds[1], stds[2]**2]] + + self.p_q_cov_matrix = [[stds[0]**2, self.p_q_corr*stds[0]*stds[1]], + [self.p_q_corr*stds[1]*stds[0], stds[1]**2]] @@ -392,8 +397,8 @@ def InternalConfigure(self, config_dict): # Parameters can be constrained manually or by inlcuding them in the nuisance parameter dictionary #self.constrained_parameter_names = reader.read_param(config_dict, 'constrained_parameter_names', []) self.constrained_parameters = reader.read_param(config_dict, 'constrained_parameters', []) - self.constrained_means = list(np.array(self.model_parameter_means)[self.constrained_parameters])#reader.read_param(config_dict, 'constrained_means', []) - self.constrained_widths = list(np.array(self.model_parameter_widths)[self.constrained_parameters])#reader.read_param(config_dict, 'constrained_widths', []) + self.constrained_means = np.array(self.model_parameter_means)[self.constrained_parameters]#reader.read_param(config_dict, 'constrained_means', []) + self.constrained_widths = np.array(self.model_parameter_widths)[self.constrained_parameters]#reader.read_param(config_dict, 'constrained_widths', []) self.nuisance_parameters = reader.read_param(config_dict, 'nuisance_parameters', {}) @@ -402,11 +407,18 @@ def InternalConfigure(self, config_dict): if p in self.nuisance_parameters.keys(): if self.nuisance_parameters[p]: self.constrained_parameters.append(i) - self.constrained_means.append(self.model_parameter_means[i]) - self.constrained_widths.append(self.model_parameter_widths[i]) + self.constrained_means = np.append(self.constrained_means, self.model_parameter_means[i]) + self.constrained_widths = np.append(self.constrained_widths, self.model_parameter_widths[i]) if i in self.constrained_parameters: self.fixed_parameters[i] = False + if self.correlated_p_q: + self.correlated_parameters = [self.scatter_peak_ratio_p_index, self.scatter_peak_ratio_q_index] + self.cov_matrix = self.p_q_cov_matrix + else: + self.correlated_parameters = [] + + # MC uncertainty propagation does not need the fit uncertainties returned by iminuit. uncertainties are instead obtained from the distribution of fit results. # But if the uncertainty is unstead propagated by adding constraiend nuisance parameters then the fit uncertainties are needed. @@ -614,6 +626,7 @@ def ConfigureFit(self): self.parameter_errors = [max([0.1, p]) for p in self.model_parameter_widths] + # else: # logger.warning('Efficiency tilt will be fitted') # self.tilted_efficiency = True @@ -733,39 +746,39 @@ def SamplePriors(self, sampled_parameters): if 'scatter_peak_ratio' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio']: self.scatter_peak_ratio_p = self.Beta_sample(self.scatter_peak_ratio_mean, self.scatter_peak_ratio_width) self.scatter_peak_ratio_q = 1 - self.fix_scatter_ratio_b = True - self.fix_scatter_ratio_c = True + #self.fix_scatter_ratio_b = True + #self.fix_scatter_ratio_c = True self.parameter_samples['scatter_peak_ratio'] = self.scatter_peak_ratio_p sample_values.append(self.scatter_peak_ratio_p) - if self.correlated_p_q_scale and 'scatter_peak_ratio_p' in sampled_parameters.keys() and 'scatter_peak_ratio_q' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_p'] and sampled_parameters['scatter_peak_ratio_q']: - logger.info('Correlated b, c, scale sampling') - correlated_vars = np.random.multivariate_normal([self.scatter_peak_ratio_p_mean, self.scatter_peak_ratio_q_mean, self.scale_mean], self.b_c_scale_cov_matrix) + if self.correlated_p_q and 'scatter_peak_ratio_p' in sampled_parameters.keys() and 'scatter_peak_ratio_q' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_p'] and sampled_parameters['scatter_peak_ratio_q']: + logger.info('Correlated p, q, scale sampling') + correlated_vars = np.random.multivariate_normal([self.scatter_peak_ratio_p_mean, self.scatter_peak_ratio_q_mean], self.p_q_cov_matrix) self.scatter_peak_ratio_p = correlated_vars[0] self.scatter_peak_ratio_q = correlated_vars[1] - self.width_scaling = correlated_vars[2] + #self.width_scaling = correlated_vars[2] - self.fix_scatter_ratio_b = True + #self.fix_scatter_ratio_b = True self.parameter_samples['scatter_peak_ratio_p'] = self.scatter_peak_ratio_p sample_values.append(self.scatter_peak_ratio_p) self.parameter_samples['scatter_peak_ratio_q'] = self.scatter_peak_ratio_q sample_values.append(self.scatter_peak_ratio_q) - self.fix_scatter_ratio_c = True + #self.fix_scatter_ratio_c = True else: if 'scatter_peak_ratio_p' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_p']: logger.info('Uncorrelated b, c, scale sampling') self.scatter_peak_ratio_p = self.Gamma_sample(self.scatter_peak_ratio_p_mean, self.scatter_peak_ratio_p_width) - self.fix_scatter_ratio_b = True + #self.fix_scatter_ratio_b = True self.parameter_samples['scatter_peak_ratio_p'] = self.scatter_peak_ratio_p sample_values.append(self.scatter_peak_ratio_p) if 'scatter_peak_ratio_q' in sampled_parameters.keys() and sampled_parameters['scatter_peak_ratio_q']: self.scatter_peak_ratio_q = self.Gamma_sample(self.scatter_peak_ratio_q_mean, self.scatter_peak_ratio_q_width) self.parameter_samples['scatter_peak_ratio_q'] = self.scatter_peak_ratio_q sample_values.append(self.scatter_peak_ratio_q) - self.fix_scatter_ratio_c = True + #self.fix_scatter_ratio_c = True if 'B' in sampled_parameters.keys() and sampled_parameters['B']: self.B = self.Gaussian_sample(self.B_mean, self.B_width) @@ -1002,34 +1015,6 @@ def beta_rates(self, K, Q, m_nu_squared):#, index): return spectrum - # def chopped_approximate_spectrum(self, E, Q, m_nu_squared): - - # if self.use_final_states: - # N_states = len(self.final_state_array[0]) - # Q_states = Q+self.final_state_array[0]-np.max(self.final_state_array[0]) - # approximate_e_phase_space = self.ephasespace(E, Q) - - # beta_rates_array = [self.beta_rates(E, Q_states[i], m_nu_squared)#, index[i]) - # * self.final_state_array[1][i] - # * approximate_e_phase_space for i in range(N_states)] - - # spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) - - # else: - # approximate_e_phase_space = self.ephasespace(E, Q) - # beta_rates_array = self.beta_rates(E, Q, m_nu_squared) * approximate_e_phase_space - # spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3) * beta_rates_array - - # channel_a_index = np.where((Eself.channel_energy_edges[0][1])) - # channel_b_index = np.where((Eself.channel_energy_edges[1][1])) - # channel_c_index = np.where((Eself.channel_energy_edges[2][1])) - - # spectrum[channel_a_index] = spectrum[channel_a_index]*self.channel_relative_livetimes[0] - # spectrum[channel_b_index] = spectrum[channel_b_index]*self.channel_relative_livetimes[1] - # spectrum[channel_c_index] = spectrum[channel_c_index]*self.channel_relative_livetimes[2] - - # return spectrum - def approximate_spectrum(self, E, Q, m_nu_squared=0): """ @@ -1049,9 +1034,9 @@ def approximate_spectrum(self, E, Q, m_nu_squared=0): * self.final_state_array[1][i] for i in range(N_states)] - spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1]) - + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1])#*self.ephasespace(E, Q) + # convolve final state spectrum # max_energy = self.max_final_state_energy_loss # dE = self.energies[1]-self.energies[0]#E[1]-E[0] # n_dE = round(max_energy/dE) @@ -1060,9 +1045,7 @@ def approximate_spectrum(self, E, Q, m_nu_squared=0): # spectrum = convolve(spectrum_nomfs, self.final_states_interp(e_lineshape), mode='same') else: - - #approximate_e_phase_space = self.ephasespace(E, Q) - spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.beta_rates(E, Q, m_nu_squared) + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.beta_rates(E, Q, m_nu_squared)#*self.ephasespace(E, Q) if self.use_relative_livetime_correction: # scale spectrum in frequency ranges according to channel livetimes diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 69a5dbec..cb1e7ec8 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -230,7 +230,7 @@ def InternalConfigure(self, params): 'sigma_array': self.sigma_array, # This is an important parameter which determines how finely resolved the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to be increased for larger datasets. - 'num_points_in_std_array':4096, #35846, + 'num_points_in_std_array': 35846, 'base_shape': 'dirac', 'path_to_osc_strengths_files': self.path_to_osc_strengths_files, 'path_to_scatter_spectra_file':self.path_to_scatter_spectra_file, From 55e2c321b8831bdbcf2cd98c99401fc5be83a061 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Sat, 28 May 2022 23:31:44 -0500 Subject: [PATCH 163/199] frequency dependence in asimov data and faster final states --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 9598cb7a..de85209e 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -659,12 +659,19 @@ def ConfigureFit(self): def SampleConvertAndFit(self, sampled_parameters={}, params= []): + # for systematic MC uncertainty propagation: Generate Asimov data to fit with random model if self.use_asimov: #temp = self.error_scaling #self.error_scaling = 0 #self._bin_efficiency, self._bin_efficiency_error = self.Efficiency(self.bin_centers, pseudo=True) - self.hist = self.TritiumSpectrumBackground(self.bin_centers, *params) + if 'frequency_dependent_response_in_asimov' in sampled_parameters.keys() and sampled_parameters['frequency_dependent_response_in_asimov']: + self.use_frequency_dependent_lineshape = True + self.SampleFrequencyDependence() + self.hist = self.TritiumSpectrumBackground(self.bin_centers, *params) + self.use_frequency_dependent_lineshape = False + else: + self.hist = self.TritiumSpectrumBackground(self.bin_centers, *params) #self.error_scaling = temp # sample priors that are to be sampled @@ -790,8 +797,7 @@ def SamplePriors(self, sampled_parameters): self.fix_endpoint = True sample_values.append(self.endpoint) if 'frequency_dependent_response' in sampled_parameters.keys() and sampled_parameters['frequency_dependent_response']: - self.p_factors = self.Gaussian_sample(self.p_factors_mean, self.p_factors_width) - self.q_factors = self.Gaussian_sample(self.q_factors_mean, self.q_factors_width) + self.SampleFrequencyDependence() sample_values.extend([np.mean(self.p_factors), np.mean(self.q_factors)]) self.use_frequency_dependent_lineshape = True @@ -803,6 +809,9 @@ def SamplePriors(self, sampled_parameters): return self.parameter_samples + def SampleFrequencyDependence(self): + self.p_factors = self.Gaussian_sample(self.p_factors_mean, self.p_factors_width) + self.q_factors = self.Gaussian_sample(self.q_factors_mean, self.q_factors_width) # ========================================================================= # Frequency - Energy conversion @@ -1000,13 +1009,13 @@ def beta_rates(self, K, Q, m_nu_squared):#, index): nu_mass_shape_squared = (Q - K)**2 -m_nu_squared index = np.where((nu_mass_shape_squared>0) & (Q_minus_K>0)) nu_mass_shape = np.sqrt(nu_mass_shape_squared[index]) - spectrum[index] = (Q_minus_K[index])*nu_mass_shape*self.ephasespace(K[index], Q) + spectrum[index] = (Q_minus_K[index])*nu_mass_shape#*self.ephasespace(K[index], Q) else: # mainz shape for negative mbeta**2 k_squared = -m_nu_squared mu = 0.66*np.sqrt(k_squared) index = np.where(Q_minus_K+mu>0) - spectrum[index] = (Q_minus_K[index]+mu*np.exp(-1-Q_minus_K[index]/mu))*np.sqrt(Q_minus_K[index]**2+k_squared)*self.ephasespace(K[index], Q) + spectrum[index] = (Q_minus_K[index]+mu*np.exp(-1-Q_minus_K[index]/mu))*np.sqrt(Q_minus_K[index]**2+k_squared)#*self.ephasespace(K[index], Q) #else: # # lanl shape for negative mbeta**2 # k_squared = -m_nu_squared @@ -1034,7 +1043,7 @@ def approximate_spectrum(self, E, Q, m_nu_squared=0): * self.final_state_array[1][i] for i in range(N_states)] - spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1])#*self.ephasespace(E, Q) + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*np.nansum(beta_rates_array, axis=0)/np.nansum(self.final_state_array[1])*self.ephasespace(E, Q) # convolve final state spectrum # max_energy = self.max_final_state_energy_loss @@ -1045,7 +1054,7 @@ def approximate_spectrum(self, E, Q, m_nu_squared=0): # spectrum = convolve(spectrum_nomfs, self.final_states_interp(e_lineshape), mode='same') else: - spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.beta_rates(E, Q, m_nu_squared)#*self.ephasespace(E, Q) + spectrum = GF**2.*Vud**2*Mnuc2/(2.*np.pi**3)*self.beta_rates(E, Q, m_nu_squared)*self.ephasespace(E, Q) if self.use_relative_livetime_correction: # scale spectrum in frequency ranges according to channel livetimes From f92ae938172faa0d9342803003888c9504e6f041 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Sun, 29 May 2022 09:22:41 -0500 Subject: [PATCH 164/199] fix for when response is not scattered --- mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index de85209e..fd494056 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -412,7 +412,7 @@ def InternalConfigure(self, config_dict): if i in self.constrained_parameters: self.fixed_parameters[i] = False - if self.correlated_p_q: + if self.is_scattered and self.correlated_p_q: self.correlated_parameters = [self.scatter_peak_ratio_p_index, self.scatter_peak_ratio_q_index] self.cov_matrix = self.p_q_cov_matrix else: From 3564ad73729be1fb146b10f413dbccaec727066b Mon Sep 17 00:00:00 2001 From: cclaessens Date: Mon, 30 May 2022 15:55:48 -0700 Subject: [PATCH 165/199] fixed correlated parameters not working in contour plot --- mermithid/processors/Fitters/BinnedDataFitter.py | 4 ++-- mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index f1c4b999..4a15de68 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -233,8 +233,8 @@ def negPoissonLogLikelihood(self, params): constrained_indices = np.in1d(self.constrained_parameters, self.correlated_parameters).nonzero()[0] param_indices = self.correlated_parameters neg_ll += 0.5*(np.log(np.linalg.det(self.cov_matrix)) + \ - np.dot(np.transpose(np.subtract(params[param_indices], self.constrained_means[constrained_indices])), \ - np.dot(np.linalg.inv(self.cov_matrix), np.subtract(params[param_indices], self.constrained_means[constrained_indices]))) + \ + np.dot(np.transpose(np.subtract(params[param_indices], np.array(self.constrained_means)[constrained_indices])), \ + np.dot(np.linalg.inv(self.cov_matrix), np.subtract(params[param_indices], np.array(self.constrained_means)[constrained_indices]))) + \ dim*np.log(2*np.pi)) return neg_ll diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index fd494056..53532a47 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -314,7 +314,7 @@ def InternalConfigure(self, config_dict): self.B_index = self.model_parameter_names.index('B') self.fixed_parameters[self.m_beta_index] = not self.fit_nu_mass self.endpoint=reader.read_param(config_dict, 'true_endpoint', 18.573e3) - logger.info('Tritium model parameters: {}'.format(np.array(self.model_parameter_names)[self.tritium_model_indices])) + #logger.info('Tritium model parameters: {}'.format(np.array(self.model_parameter_names)[self.tritium_model_indices])) # final state spectrum From 1621d29e8ac7a3296177ddfb5e7ef373d1164235 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Tue, 31 May 2022 14:35:02 -0700 Subject: [PATCH 166/199] h2_fractionn can be MC propagated --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 53532a47..deae9f26 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -173,7 +173,6 @@ def InternalConfigure(self, config_dict): self.lineshape_model = reader.read_param(config_dict, 'lineshape_model', 'simplified') self.simplified_lineshape_path = reader.read_param(config_dict, 'simplified_lineshape_path', "required") self.helium_lineshape_path = reader.read_param(config_dict, 'helium_lineshape_path', "optional") - self.hydrogen_proportion = reader.read_param(config_dict, 'hydrogen_proportion', 1) self.use_helium_scattering = reader.read_param(config_dict, 'use_helium_scattering', False) self.use_frequency_dependent_lineshape = reader.read_param(config_dict, 'use_frequency_dependent_lineshape', True) self.correlated_p_q_scale = reader.read_param(config_dict, 'correlated_p_q_scale', False) @@ -282,12 +281,15 @@ def InternalConfigure(self, config_dict): self.scatter_peak_ratio_p_width = reader.read_param(config_dict, 'scatter_peak_ratio_p_width', 0.1) self.scatter_peak_ratio_q_mean = reader.read_param(config_dict, 'scatter_peak_ratio_q_mean', 0.7) self.scatter_peak_ratio_q_width = reader.read_param(config_dict, 'scatter_peak_ratio_q_width', 0.1) + self.h2_fraction_mean = reader.read_param(config_dict, 'h2_fraction_mean', 1.0) + self.h2_fraction_width = reader.read_param(config_dict, 'h2_fraction_width', 0.0) self.scatter_peak_ratio_mean = reader.read_param(config_dict, 'scatter_peak_ratio_mean', 0.5) self.scatter_peak_ratio_width = reader.read_param(config_dict, 'scatter_peak_ratio_width', 0.1) self.scatter_peak_ratio_p = self.scatter_peak_ratio_p_mean self.scatter_peak_ratio_q = self.scatter_peak_ratio_q_mean + self.h2_fraction = self.h2_fraction_mean #Adding correlated parameters (will later incorporate into morpho processor) self.p_q_corr = reader.read_param(config_dict, 'p_q_corr', 0) @@ -497,7 +499,7 @@ def InternalConfigure(self, config_dict): 'max_scatters': self.NScatters, 'fix_scatter_proportion': True, # When fix_scatter_proportion is True, set the scatter proportion for gas1 below - 'gas1_scatter_proportion': self.hydrogen_proportion, + 'gas1_scatter_proportion': self.h2_fraction, # This is an important parameter which determines how finely resolved # the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to # be increased for larger datasets. @@ -742,6 +744,12 @@ def SamplePriors(self, sampled_parameters): # self.res = 30.01/float(2*np.sqrt(2*np.log(2))) self.parameter_samples['resolution'] = self.res sample_values.append(self.res) + if 'h2_fraction' in sampled_parameters.keys() and sampled_parameters['h2_fraction']: + self.h2_fraction = self.Gaussian_sample(self.h2_fraction_mean, self.h2_fraction_width) + if self.h2_fraction > 1: self.h2_fraction=1 + elif self.h2_fraction < 0: self.h2_fraction=0 + self.parameter_samples['h2_fraction'] = self.h2_fraction + sample_values.append(self.h2_fraction) if 'two_gaussian_sigma_1' in sampled_parameters.keys() and sampled_parameters['two_gaussian_sigma_1']: self.two_gaussian_sigma_1 = self.Gaussian_sample(self.two_gaussian_sigma_1_mean, self.two_gaussian_sigma_1_width) self.parameter_samples['two_gaussian_sigma_1'] = self.two_gaussian_sigma_1 @@ -1127,12 +1135,12 @@ def f_i(self, K, gamma, mu, sigma): - def more_accurate_simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1): + def more_accurate_simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1, h2_fraction=1): """ Still a simplified lineshape but helium is not just a gaussian """ if self.plot_lineshape: - logger.info('Using more accurate multi gas scattering. Hydrogen proportion is {}'.format(self.hydrogen_proportion)) + logger.info('Using more accurate multi gas scattering. Hydrogen proportion is {}'.format(self.h2_fraction)) p0, p1, p2, p3 = self.lineshape_p[1], self.lineshape_p[3], self.lineshape_p[5], self.lineshape_p[7] @@ -1191,8 +1199,8 @@ def more_accurate_simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, #norm_h = np.sum(shape0 + hydrogen_scattering) #norm_he = np.sum(shape0 + helium_scattering) - lineshape = (self.hydrogen_proportion*(shape0 + hydrogen_scattering)/norm_h + - (1-self.hydrogen_proportion)*(shape0 + helium_scattering)/norm_he) + lineshape = (h2_fraction*(shape0 + hydrogen_scattering)/norm_h + + (1-h2_fraction)*(shape0 + helium_scattering)/norm_he) #plt.plot(K, lineshape/np.max(lineshape), color='darkgreen', label='full: {} hydrogen'.format(self.hydrogen_proportion)) #plt.xlim(-200, 200) @@ -1206,7 +1214,7 @@ def simplified_multi_gas_lineshape(self, K, Kcenter, FWHM, prob_b, prob_c=1, h2_ """ if self.plot_lineshape: - logger.info('Using simplified lineshape. Hydrogen proportion is {}'.format(self.hydrogen_proportion)) + logger.info('Using simplified lineshape. Hydrogen proportion is {}'.format(h2_fraction)) p0, p1, p2, p3 = self.lineshape_p[1], self.lineshape_p[3], self.lineshape_p[5], self.lineshape_p[7] q0, q1, q2, q3 = self.helium_lineshape_p[1], self.helium_lineshape_p[3], self.helium_lineshape_p[5], self.helium_lineshape_p[7] @@ -1426,7 +1434,7 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p if 'h2_fraction' in self.model_parameter_names: h2_fraction = args[self.h2_fraction_index] else: - h2_fraction = self.hydrogen_proportion + h2_fraction = self.h2_fraction # simplified lineshape FWHM = 2.*np.sqrt(2.*np.log(2.))*res *self.width_scaling From 36934ab87d0f974d97fec40e95b2801e6c95575c Mon Sep 17 00:00:00 2001 From: Talia Weiss Date: Mon, 27 Jun 2022 17:12:46 -0400 Subject: [PATCH 167/199] Updated molecular endpoint value --- mermithid/misc/Constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mermithid/misc/Constants.py b/mermithid/misc/Constants.py index a967fae4..5705b850 100644 --- a/mermithid/misc/Constants.py +++ b/mermithid/misc/Constants.py @@ -32,8 +32,8 @@ def GF(): return 1.1663787*10**(-23) #Gf/(hc)^3, in eV^(-2) def Vud(): return 0.97425 #CKM element #Beta decay-specific physical constants -def QT(): return 18563.251 #For atomic tritium (eV), from Bodine et al. (2015) -def QT2(): return 18573.24 #For molecular tritium (eV), Bodine et al. (2015) +def QT(): return 18563.251 #SHOULD BE DOUBLE-CHECKED. For atomic tritium (eV), from Bodine et al. (2015) +def QT2(): return 18574.01 #For molecular tritium (eV). Calculation here: https://projecteight.slack.com/archives/CG5TY2UE7/p1649963449399179, based on Bodine et al. (2015). def Rn(): return 2.8840*10**(-3) #Helium-3 nuclear radius in units of me, from Kleesiek et al. (2018): https://arxiv.org/pdf/1806.00369.pdf def M_3He_in_me(): return 5497.885 #Helium-3 mass in units of me, Kleesiek et al. (2018) def atomic_num(): return 2. #For helium-3 From 20799ef904c2cc0b58af5b779a55763b057edda9 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Tue, 5 Jul 2022 14:19:13 -0700 Subject: [PATCH 168/199] working on neutrino mass fit fix --- .../processors/Fitters/BinnedDataFitter.py | 98 ++++++++++++++----- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 23 +++-- 2 files changed, 89 insertions(+), 32 deletions(-) diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index 4a15de68..0e6ec4ac 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -11,9 +11,10 @@ from __future__ import absolute_import import numpy as np -import scipy -import sys -import json +#import scipy +from scipy.optimize import NonlinearConstraint +#import sys +#import json from iminuit import Minuit from morpho.utilities import morphologging, reader from morpho.processors import BaseProcessor @@ -68,6 +69,7 @@ def InternalConfigure(self, params): self.cov_matrix = np.array(reader.read_param(params, 'covariance_matrix', [])) self.minos_cls = reader.read_param(params, 'minos_confidence_level_list', []) self.minos_intervals = reader.read_param(params,'find_minos_intervals', False) + self.free_in_second_fit = reader.read_param(params, 'free_in_second_fit', []) # derived configurations self.bin_centers = self.bins[0:-1]+0.5*(self.bins[1]-self.bins[0]) @@ -121,9 +123,12 @@ def setup_minuit(self): self.m_binned.errordef = 0.5 self.m_binned.errors = self.parameter_errors - self.m_binned.throw_nan = True + self.m_binned.throw_nan = False + self.m_binned.strategy = 1 self.m_binned.print_level = self.print_level + self.constraints = [] + for i, name in enumerate(self.parameter_names): if name in self.fixes_dict.keys(): self.m_binned.fixed[name]= self.fixes_dict[name] @@ -131,7 +136,8 @@ def setup_minuit(self): else: self.m_binned.fixed[name] = self.fixes[i] self.m_binned.limits[name] = self.limits[i] - + if not all(l is None for l in self.limits[i]): + self.constraints.append(NonlinearConstraint(lambda x: x, self.limits[i][0], self.limits[i][1])) def fit(self): @@ -153,38 +159,64 @@ def fit(self): self.setup_minuit() # minimze - self.m_binned.simplex().migrad() - #m_binned.hesse() + self.m_binned.simplex().migrad() #scipy(constraints=self.constraints) + if len(self.free_in_second_fit) > 0: + for p in self.free_in_second_fit: + self.m_binned.fixed[p] = False + logger.info('{} now free. Minimize again'.format(self.free_in_second_fit)) + self.m_binned.migrad() + self.m_binned.hesse() #m_binned.minos() #self.param_states = m_binned.get_param_states() - #logger.info(self.param_states) + if self.print_level: + logger.info(self.m_binned.params) + logger.info(self.m_binned.values) + logger.info(self.m_binned.errors) # results result_array = np.array(self.m_binned.values) error_array = np.array(self.m_binned.errors) + if self.minos_intervals: self.minos_errors = {} for mcl in self.minos_cls: logger.info('Getting minos errors for CL = {}'.format(mcl)) try: self.m_binned.minos(cl=mcl) + + self.minos_errors[mcl] = {} + if self.print_level: + logger.info(self.m_binned.params) + logger.info(self.m_binned.merrors) + for k in self.m_binned.merrors.keys(): + self.minos_errors[mcl][k] = {'interval': [self.m_binned.merrors[k].lower, self.m_binned.merrors[k].upper], + 'number': self.m_binned.merrors[k].number, + 'name': self.m_binned.merrors[k].name, + 'is_valid': self.m_binned.merrors[k].is_valid and self.m_binned.valid} + except RuntimeError as e: print(self.m_binned.params) - raise e - self.minos_errors[mcl] = {} - if self.print_level: - logger.info(self.m_binned.merrors) - for k in self.m_binned.merrors.keys(): - self.minos_errors[mcl][k] = {'interval': [self.m_binned.merrors[k].lower, self.m_binned.merrors[k].upper], - 'number': self.m_binned.merrors[k].number, - 'name': self.m_binned.merrors[k].name, - 'is_valid': self.m_binned.merrors[k].is_valid} + print(self.m_binned.merrors) + logger.error('Minos failed. Returning Hesse instead') + for i in range(len(self.parameter_names)): + self.minos_errors[mcl][k] = {'interval': [-self.m_binned.errors[k], self.m_binned.errors[k]], + 'number': self.m_binned.merrors[k].number, + 'name': self.m_binned.merrors[k].name, + 'is_valid': False} + #raise e + + else: + self.hesse_errors = {0.683: {}} + for i in range(len(self.parameter_names)): + self.hesse_errors[0.683][self.parameter_names[i]] = {'interval': [-error_array[i], error_array[i]], + 'number': i, + 'name': self.parameter_names[i], + 'is_valid': self.m_binned.valid} if self.print_level == 1: - logger.info('Fit results: {}'.format(result_array)) - logger.info('Errors: {}'.format(error_array)) + logger.info(self.m_binned.fmin) #logger.info('Correlation matrix: {}'.format(self.m_binned.covariance.correlation())) return result_array, error_array @@ -200,12 +232,29 @@ def PoissonLogLikelihood(self, params): else: expectation = model_return - if np.min(expectation) < 0: - logger.error('Expectation contains negative numbers: Minimum {} -> {}. They will be excluded but something could be horribly wrong.'.format(np.argmin(expectation), np.min(expectation))) - logger.error('FYI, the parameters are: {}'.format(params)) + # if np.min(expectation) < 0: + # logger.error('Expectation contains negative numbers: Minimum {} -> {}.'.format(np.argmin(expectation), np.min(expectation))) + # logger.error('FYI, the parameters are: {}'.format(params)) + # logger.info('Expectation is: {}'.format(expectation)) + # import matplotlib.pyplot as plt + # plt.figure() + # plt.step(self.bin_centers, self.hist) + # plt.plot(self.bin_centers, expectation) + + # #raise ValueError('Expecation below zero') + + # logger.error("try again") + # # expectation + # model_return = self.model(self.bin_centers, *params) + # if np.shape(model_return)[0] == 2: + # expectation, expectation_error = model_return + # else: + # expectation = model_return + # plt.plot(self.bin_centers, expectation) + # plt.savefig('negative_expectation.png', dpi=200) # exclude bins where expectation is <= zero or nan - index = np.where(expectation>0)#np.finfo(0.0).resolution) + index = np.where((expectation>np.min(expectation)))#np.where(expectation>0)#np.finfo(0.0).resolution) # poisson log likelihoood ll = (self.hist[index]*np.log(expectation[index]) - expectation[index]).sum() @@ -228,7 +277,8 @@ def negPoissonLogLikelihood(self, params): if not param in self.correlated_parameters: neg_ll += 0.5 * ((params[param] - self.constrained_means[i])/ self.constrained_widths[i])**2 + 0.5*np.log(2*np.pi) + np.log(self.constrained_widths[i]) - if len(self.correlated_parameters) > 0: + constrained_indices = np.in1d(self.constrained_parameters, self.correlated_parameters).nonzero()[0] + if len(constrained_indices) > 0: dim = len(self.correlated_parameters) constrained_indices = np.in1d(self.constrained_parameters, self.correlated_parameters).nonzero()[0] param_indices = self.correlated_parameters diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index deae9f26..81d93fef 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -118,9 +118,10 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, total_counts = results[1]+results[3] if fit_config_dict['minos_intervals']: + print(T.m_binned.covariance.correlation()) return results, T.minos_errors, total_counts else: - return results, errors, total_counts + return results, T.hesse_errors, total_counts def GetPDF(fit_config_dict, params, plot=False): @@ -191,6 +192,7 @@ def InternalConfigure(self, config_dict): self.model_parameter_widths = reader.read_param(config_dict, 'model_parameter_widths', [100, 0.1, 0.1, 500, 0.1, 0, 3, 1, 1]) self.fixed_parameters = reader.read_param(config_dict, 'fixed_parameters', [False, False, False, False, True, True, True, True, True]) self.fixed_parameter_dict = reader.read_param(config_dict, 'fixed_parameter_dict', {}) + self.free_in_second_fit = reader.read_param(config_dict, 'free_in_second_fit', []) self.limits = reader.read_param(config_dict, 'model_parameter_limits', [[18e3, 20e3], [0, None], @@ -1008,7 +1010,7 @@ def derived_two_gaussian_resolution(self, energy_array, sigma_s, mu_1, mu_2, A=1 return lineshape - def beta_rates(self, K, Q, m_nu_squared):#, index): + def beta_rates(self, K, Q, m_nu_squared, shape='mainz'):#, index): spectrum = np.zeros(len(K)) Q_minus_K = Q-K @@ -1018,17 +1020,22 @@ def beta_rates(self, K, Q, m_nu_squared):#, index): index = np.where((nu_mass_shape_squared>0) & (Q_minus_K>0)) nu_mass_shape = np.sqrt(nu_mass_shape_squared[index]) spectrum[index] = (Q_minus_K[index])*nu_mass_shape#*self.ephasespace(K[index], Q) - else: + elif shape=='mainz': # mainz shape for negative mbeta**2 k_squared = -m_nu_squared mu = 0.66*np.sqrt(k_squared) index = np.where(Q_minus_K+mu>0) spectrum[index] = (Q_minus_K[index]+mu*np.exp(-1-Q_minus_K[index]/mu))*np.sqrt(Q_minus_K[index]**2+k_squared)#*self.ephasespace(K[index], Q) - #else: - # # lanl shape for negative mbeta**2 - # k_squared = -m_nu_squared - # index = np.where(Q_minus_K > 0) - # spectrum[index] = Q_minus_K[index]**2+k_squared/2 + elif shape=='lanl': + # lanl shape for negative mbeta**2 + k_squared = -m_nu_squared + index = np.where(Q_minus_K > 0) + spectrum[index] = Q_minus_K[index]**2+k_squared/2 + elif shape=='llnl': + #print('llnl') + k_squared = -m_nu_squared + index = np.where(Q_minus_K+np.sqrt(k_squared)>0) + spectrum[index] = np.abs((Q_minus_K[index])**2+k_squared*Q_minus_K[index]/(2*np.abs(Q_minus_K[index]))) return spectrum From 9bb26f4b11aab42701734a01d5727052ac403dd4 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Tue, 23 Aug 2022 16:26:16 -0700 Subject: [PATCH 169/199] updates in fake data generation and likelihood fitter. new function in fake data generator to first randomize, then interpolate efficiencies. fake data generator only samples resolution files once. Binned data fitter can use chi square minimization instead of MLL. Option to repeat fit with new fixed and free parameters. If expectation contains negative numbers the model is plotted and expectation is calculated a second time. Lots of commented code. Clean later. --- mermithid/misc/FakeTritiumDataFunctions.py | 14 ++ .../processors/Fitters/BinnedDataFitter.py | 128 +++++++++++++----- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 15 +- .../TritiumSpectrum/FakeDataGenerator.py | 8 +- .../misc/MultiGasComplexLineShape.py | 28 +++- 5 files changed, 148 insertions(+), 45 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index df9999fe..fbb72516 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -441,3 +441,17 @@ def efficiency_from_interpolation(x, efficiency_dict, B=0.9578186017836624): +def random_efficiency_from_interpolation(x, efficiency_dict, B=0.9578186017836624): + """ + Function to calculate efficiency + """ + logger.info('Sampling efficiencies before interpolation') + f = Frequency(x, B) + + efficiency_mean = efficiency_dict['eff interp with slope correction'] + efficiency_error = np.mean(efficiency_dict['error interp with slope correction'], axis=0) + random_efficiencies = np.random.normal(efficiency_mean, efficiency_error) + random_efficiencies[random_efficiencies<0] = 0. + interp_efficiency = interp1d(efficiency_dict['frequencies'], random_efficiencies, fill_value='0', bounds_error=False) + + return interp_efficiency(f) \ No newline at end of file diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index 0e6ec4ac..c35fd74d 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -16,8 +16,10 @@ #import sys #import json from iminuit import Minuit +from iminuit.cost import LeastSquares from morpho.utilities import morphologging, reader from morpho.processors import BaseProcessor +from scipy.stats import chisquare logger = morphologging.getLogger(__name__) @@ -70,6 +72,8 @@ def InternalConfigure(self, params): self.minos_cls = reader.read_param(params, 'minos_confidence_level_list', []) self.minos_intervals = reader.read_param(params,'find_minos_intervals', False) self.free_in_second_fit = reader.read_param(params, 'free_in_second_fit', []) + self.fixed_in_second_fit = reader.read_param(params, 'fixed_in_second_fit', []) + self.error_def = reader.read_param(params,'error_def', 1) # derived configurations self.bin_centers = self.bins[0:-1]+0.5*(self.bins[1]-self.bins[0]) @@ -100,7 +104,7 @@ def InternalRun(self): self.results['param_errors'] = error_array self.results['correlation_matrix'] = np.array(self.m_binned.covariance.correlation()) for i, k in enumerate(self.parameter_names): - self.results[k] = {'value': result_array[i], 'error': error_array[i]} + self.results[k] = {'value': result_array[i], 'error': error_array[i], 'likelihood': self.likelihood} return True @@ -116,12 +120,22 @@ def model(self, x, A, mu, sigma): def setup_minuit(self): - self.m_binned = Minuit(self.negPoissonLogLikelihood, - self.initial_values, - name=self.parameter_names, - ) + if self.error_def == 0.5: + self.m_binned = Minuit(self.negPoissonLogLikelihood, + self.initial_values, + name=self.parameter_names, + ) + + self.m_binned.errordef = 0.5 + logger.info('Doing MLL analysis') + else: + #least_squares = LeastSquares(self.bin_centers, self.hist, np.sqrt(self.hist), self.model) + self.m_binned = Minuit(self.leastSquares, + self.initial_values, + name=self.parameter_names) + self.m_binned.errordef = 1.0 + logger.info('Doing chi square analysis') - self.m_binned.errordef = 0.5 self.m_binned.errors = self.parameter_errors self.m_binned.throw_nan = False self.m_binned.strategy = 1 @@ -136,12 +150,15 @@ def setup_minuit(self): else: self.m_binned.fixed[name] = self.fixes[i] self.m_binned.limits[name] = self.limits[i] - if not all(l is None for l in self.limits[i]): - self.constraints.append(NonlinearConstraint(lambda x: x, self.limits[i][0], self.limits[i][1])) + #if not all(l is None for l in self.limits[i]): + # self.constraints.append(NonlinearConstraint(lambda x: x, self.limits[i][0], self.limits[i][1])) def fit(self): # Now minimize neg log likelihood using iMinuit + self.setup_minuit() + + if self.print_level > 0: logger.info('This is the plan:') logger.info('\tFitting data consisting of {} elements'.format(np.sum(self.hist))) @@ -154,16 +171,23 @@ def fit(self): logger.info('\tConstraint means: {}'.format(self.constrained_means)) logger.info('\tConstraint widths: {}'.format(self.constrained_widths)) logger.info('\tCorrelated parameters: {}'.format([self.parameter_names[i] for i in self.correlated_parameters])) + logger.info('\tError def : {}'.format(self.m_binned.errordef)) + logger.info('\tMinos uncertainties: {}'.format(self.minos_cls)) - - self.setup_minuit() # minimze self.m_binned.simplex().migrad() #scipy(constraints=self.constraints) if len(self.free_in_second_fit) > 0: for p in self.free_in_second_fit: self.m_binned.fixed[p] = False - logger.info('{} now free. Minimize again'.format(self.free_in_second_fit)) + logger.info('{} now free.'.format(self.free_in_second_fit)) + if len(self.fixed_in_second_fit) > 0: + for p in self.fixed_in_second_fit: + self.m_binned.fixed[p] = True + #self.m_binned.limits[p] = [None, None] + logger.info('{} now fixed'.format(self.fixed_in_second_fit)) + if len(self.free_in_second_fit) > 0 or len(self.fixed_in_second_fit) > 0: + logger.info('Minimize again') self.m_binned.migrad() self.m_binned.hesse() #m_binned.minos() @@ -177,6 +201,7 @@ def fit(self): # results result_array = np.array(self.m_binned.values) error_array = np.array(self.m_binned.errors) + self.likelihood = self.PoissonLogLikelihood(result_array) if self.minos_intervals: @@ -200,11 +225,12 @@ def fit(self): print(self.m_binned.params) print(self.m_binned.merrors) logger.error('Minos failed. Returning Hesse instead') + self.minos_errors[mcl] = {} for i in range(len(self.parameter_names)): - self.minos_errors[mcl][k] = {'interval': [-self.m_binned.errors[k], self.m_binned.errors[k]], - 'number': self.m_binned.merrors[k].number, - 'name': self.m_binned.merrors[k].name, - 'is_valid': False} + self.minos_errors[mcl][self.parameter_names[i]] = {'interval': [-error_array[i], error_array[i]], + 'number': i, + 'name': self.parameter_names[i], + 'is_valid': self.m_binned.valid} #raise e else: @@ -232,29 +258,36 @@ def PoissonLogLikelihood(self, params): else: expectation = model_return - # if np.min(expectation) < 0: - # logger.error('Expectation contains negative numbers: Minimum {} -> {}.'.format(np.argmin(expectation), np.min(expectation))) - # logger.error('FYI, the parameters are: {}'.format(params)) - # logger.info('Expectation is: {}'.format(expectation)) - # import matplotlib.pyplot as plt - # plt.figure() - # plt.step(self.bin_centers, self.hist) - # plt.plot(self.bin_centers, expectation) - - # #raise ValueError('Expecation below zero') - - # logger.error("try again") - # # expectation - # model_return = self.model(self.bin_centers, *params) - # if np.shape(model_return)[0] == 2: - # expectation, expectation_error = model_return - # else: - # expectation = model_return - # plt.plot(self.bin_centers, expectation) - # plt.savefig('negative_expectation.png', dpi=200) + if np.min(expectation) < 0: + logger.error('Expectation contains negative numbers: Minimum {} -> {}.'.format(np.argmin(expectation), np.min(expectation))) + logger.error('FYI, the parameters are: {}'.format(params)) + logger.info('Expectation is: {}'.format(expectation)) + import matplotlib.pyplot as plt + plt.figure() + plt.step(self.bin_centers, self.hist) + plt.plot(self.bin_centers, expectation) + + #raise ValueError('Expecation below zero') + + logger.error("try again") + # expectation + model_return = self.model(self.bin_centers, *params) + if np.shape(model_return)[0] == 2: + expectation, expectation_error = model_return + else: + expectation = model_return + plt.plot(self.bin_centers, expectation,linestyle='--') + plt.savefig('negative_expectation.png', dpi=200) # exclude bins where expectation is <= zero or nan - index = np.where((expectation>np.min(expectation)))#np.where(expectation>0)#np.finfo(0.0).resolution) + index = np.where(expectation>0)#np.where(expectation>0)#np.finfo(0.0).resolution) + if "background" in self.parameter_names and self.m_binned.fixed["background"]: + endpoint_parameter = self.parameter_names.index('endpoint') + index = np.where((self.bin_centers+0.5*(self.bin_centers[1]-self.bin_centers[0])<=params[endpoint_parameter]) & + (self.hist>=5)) + #index = np.arange(np.min(np.where(self.hist>1)[0]), np.max(np.where(self.hist>1)[0])+1) + #print(self.hist[index]) + #print(self.bin_centers[index]) # poisson log likelihoood ll = (self.hist[index]*np.log(expectation[index]) - expectation[index]).sum() @@ -288,3 +321,26 @@ def negPoissonLogLikelihood(self, params): dim*np.log(2*np.pi)) return neg_ll + + + def leastSquares(self, params): + + # expectation + model_return = self.model(self.bin_centers, *params) + if np.shape(model_return)[0] == 2: + expectation, expectation_error = model_return + else: + expectation = model_return + + nonzero_bins_index = np.where(self.hist>0)#np.where(expectation>0)#np.finfo(0.0).resolution) + zero_bins_index = np.where(self.hist==0) + index = np.where(expectation>0) + if "background" in self.parameter_names and self.m_binned.fixed["background"]: + # endpoint_parameter = self.parameter_names.index('endpoint') + index = np.where((expectation>1) & (self.hist>1)) + total_chisquare, _ = chisquare(self.hist[index], expectation[index]) + #chi2 = np.nansum((self.hist[nonzero_bins_index]-expectation[nonzero_bins_index])**2/expectation[nonzero_bins_index]) + #chi2+= np.nansum((self.hist[zero_bins_index]-expectation[zero_bins_index])**2) + #chi2 = 2*((expectation - self.hist + self.hist*np.log(self.hist/expectation))[nonzero_bins_index]).sum() + #chi2 += 2*(expectation - self.hist)[zero_bins_index].sum() + return total_chisquare #LeastSquares(self.bin_centers, self.hist, np.sqrt(self.hist), self.model) \ No newline at end of file diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 81d93fef..a705aa87 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -193,6 +193,7 @@ def InternalConfigure(self, config_dict): self.fixed_parameters = reader.read_param(config_dict, 'fixed_parameters', [False, False, False, False, True, True, True, True, True]) self.fixed_parameter_dict = reader.read_param(config_dict, 'fixed_parameter_dict', {}) self.free_in_second_fit = reader.read_param(config_dict, 'free_in_second_fit', []) + self.fixed_in_second_fit = reader.read_param(config_dict, 'fixed_in_second_fit', []) self.limits = reader.read_param(config_dict, 'model_parameter_limits', [[18e3, 20e3], [0, None], @@ -203,7 +204,7 @@ def InternalConfigure(self, config_dict): [12, 100], [-100, 100], [-100, 100]]) - + self.error_def = reader.read_param(config_dict,'error_def', 0.5) # check that configuration is consistent if (len(self.model_parameter_names) != len(self.model_parameter_means) or len(self.model_parameter_names) != len(self.model_parameter_widths) or len(self.model_parameter_names) != len(self.fixed_parameters)): @@ -1036,6 +1037,15 @@ def beta_rates(self, K, Q, m_nu_squared, shape='mainz'):#, index): k_squared = -m_nu_squared index = np.where(Q_minus_K+np.sqrt(k_squared)>0) spectrum[index] = np.abs((Q_minus_K[index])**2+k_squared*Q_minus_K[index]/(2*np.abs(Q_minus_K[index]))) + elif shape=='katrin': + k_squared = -m_nu_squared + index = np.where(Q_minus_K>0) + spectrum[index] = Q_minus_K[index]*np.sqrt((Q_minus_K[index])**2+k_squared) + #print(Q_minus_K) + #print(len(K[self.index])) + #print(len(K)) + #print(np.max(K[index])) + #print(np.max(K)) return spectrum @@ -1400,6 +1410,7 @@ def TritiumSpectrum(self, E=[], *args, error=False):#endpoint=18.6e3, m_nu=0., p e_spec = np.arange(min(self.energies)-max_energy, max(self.energies)+max_energy, dE) + #e_spec = self.energies #np.r_[e_add, self._energies] # energy resolution @@ -1738,7 +1749,7 @@ def Efficiency(self, E, freq=False, pseudo=False): efficiency = pseudo_efficiency if np.min(efficiency) < 0: - index = np.where(E>=np.min(E[efficiency<0])) + index = np.where(efficiency < 0)#E>=np.min(E[efficiency<0])) efficiency[index] = 0. diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index cb1e7ec8..3bfe7c40 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -230,7 +230,7 @@ def InternalConfigure(self, params): 'sigma_array': self.sigma_array, # This is an important parameter which determines how finely resolved the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to be increased for larger datasets. - 'num_points_in_std_array': 35846, + 'num_points_in_std_array': 10000,#35846, 'base_shape': 'dirac', 'path_to_osc_strengths_files': self.path_to_osc_strengths_files, 'path_to_scatter_spectra_file':self.path_to_scatter_spectra_file, @@ -379,9 +379,11 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 if efficiency_dict is not None: logger.info('Evaluating efficiencies') - efficiency_mean, efficiency_error = efficiency_from_interpolation(self.Koptions, efficiency_dict, B_field) + """efficiency_mean, efficiency_error = efficiency_from_interpolation(self.Koptions, efficiency_dict, B_field) logger.info("Sampling efficiencies given means and uncertainties") - efficiency = np.random.normal(efficiency_mean, efficiency_error) + efficiency = np.random.normal(efficiency_mean, efficiency_error)""" + + efficiency = random_efficiency_from_interpolation(self.Koptions, efficiency_dict, B_field) eff_negative = (efficiency<0.) efficiency[eff_negative] = 0. #Whenever this occurs, efficiency_mean=0 and efficiency_error=1 else: diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 5fdacd22..4f37a0d9 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -118,6 +118,10 @@ def InternalConfigure(self, params): # Read shake parameters from JSON file if self.base_shape == 'shake': self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) + + # read in resolution if simulated + if 'simulated' in self.resolution_function: + self.sample_and_interpolate_resolution() return True def InternalRun(self): @@ -524,7 +528,7 @@ def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_a return normalized_convolved_spectrum def convolve_simulated_resolution_scaled(self, working_spectrum, scale_factor): - if self.use_combined_four_trap_inst_reso: + """if self.use_combined_four_trap_inst_reso: x_data, y_data, y_err_data = self.combine_four_trap_resolution_from_txt(self.trap_weights) logger.info("Combined four instrumental resolution files") else: @@ -534,15 +538,31 @@ def convolve_simulated_resolution_scaled(self, working_spectrum, scale_factor): y_data = np.random.normal(y_data, y_err_data) logger.info("Sampling instrumental resolution counts per bin") scaled_xdata = x_data*scale_factor - f = interpolate.interp1d(x_data*scale_factor, y_data) + f = interpolate.interp1d(x_data*scale_factor, y_data)""" + x_array = self.std_eV_array() y_array = np.zeros(len(x_array)) - index_within_range_of_xdata = np.where((x_array >= scaled_xdata[0]) & (x_array <= scaled_xdata[-1])) - y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]) + + #index_within_range_of_xdata = np.where((x_array >= scaled_xdata[0]) & (x_array <= scaled_xdata[-1])) + #y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]/scale_factor) + y_array = self.interpolated_resolution(x_array/scale_factor) convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum + # do resolution sampling + def sample_and_interpolate_resolution(self): + if self.use_combined_four_trap_inst_reso: + x_data, y_data, y_err_data = self.combine_four_trap_resolution_from_txt(self.trap_weights) + logger.info("Combined four instrumental resolution files") + else: + x_data, y_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + logger.info("Using ONE simulated instrumental resolution file (not combining four)") + if self.sample_ins_resolution_errors: + y_data = np.random.normal(y_data, y_err_data) + logger.info("Sampling instrumental resolution counts per bin") + self.interpolated_resolution = interpolate.interp1d(x_data, y_data, fill_value=(0,0), bounds_error=False) + #might be untested def convolve_smeared_triangle(self, func_to_convolve, center, scale1, scale2, exponent, sigma): resolution_f = self.std_smeared_triangle(center, scale1, scale2, exponent, sigma) From b972e1b8a72d8f7783a1eb91da8f694f152f3643 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Mon, 29 Aug 2022 15:08:30 -0700 Subject: [PATCH 170/199] commented freq var logging --- mermithid/misc/FakeTritiumDataFunctions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index fbb72516..f9aa3f02 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -285,6 +285,10 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, logger.info('Lineshape is {} with {}'.format(lineshape, resolution_function)) energy_half_range = max(max_energy, abs(min_energy)) + #logger.info('Using {} frequency regions. Mean and std of p are {} and {}. For q its {} and {}'.format(len(ins_res_width_bounds)-1, + # np.mean(p_factors), np.std(p_factors), + # np.mean(q_factors), np.std(q_factors))) + if ins_res_width_bounds != None: Kbounds = [np.min(K)] + ins_res_width_bounds + [np.max(K)] else: From 05399621a7c0045e972b7d9c715f92ab0d73c4be Mon Sep 17 00:00:00 2001 From: cclaessens Date: Tue, 8 Nov 2022 16:23:31 -0800 Subject: [PATCH 171/199] returning likelihoods on demand --- .../processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index a705aa87..ae2de31b 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -120,6 +120,15 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, if fit_config_dict['minos_intervals']: print(T.m_binned.covariance.correlation()) return results, T.minos_errors, total_counts + elif fit_config_dict['return_ll']: + results_best_mass = deepcopy(results) + results_best_mass[2] = max(0, results[2]) + # get likleihood from fit + fit_ll = T.PoissonLogLikelihood(results) + # get likleihood of asimov best mass + T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_best_mass) + best_ll = T.PoissonLogLikelihood(results) + return results, T.hesse_errors, [fit_ll, best_ll] else: return results, T.hesse_errors, total_counts From 042d916b7296164ab2590993e73e4f6f6fa6d370 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 9 Nov 2022 23:16:42 -0800 Subject: [PATCH 172/199] working on likelihood returns for fc limits --- mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index ae2de31b..0b31bba1 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -124,6 +124,7 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, results_best_mass = deepcopy(results) results_best_mass[2] = max(0, results[2]) # get likleihood from fit + T.hist = T.TritiumSpectrumBackground(T.bin_centers, *fit_config_dict['model_parameter_means']) fit_ll = T.PoissonLogLikelihood(results) # get likleihood of asimov best mass T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_best_mass) From 9e5354f4b5a52b6780b8d34861bbad2ba31d489f Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 10 Nov 2022 15:34:12 -0800 Subject: [PATCH 173/199] returning LL given true mass model instead of tue data --- mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 0b31bba1..2d3f3143 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -123,8 +123,10 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, elif fit_config_dict['return_ll']: results_best_mass = deepcopy(results) results_best_mass[2] = max(0, results[2]) + results_true_mass = deepcopy(results) + results_true_mass[2] = fit_config_dict['model_parameter_means'][2] # get likleihood from fit - T.hist = T.TritiumSpectrumBackground(T.bin_centers, *fit_config_dict['model_parameter_means']) + T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_true_mass) fit_ll = T.PoissonLogLikelihood(results) # get likleihood of asimov best mass T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_best_mass) From bc2aa39cf1e6088d44271e56438a514826db9ec0 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 10 Nov 2022 19:49:58 -0800 Subject: [PATCH 174/199] use neg neg likelihood to get constrained result --- .../processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 2d3f3143..13f46131 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -125,12 +125,14 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, results_best_mass[2] = max(0, results[2]) results_true_mass = deepcopy(results) results_true_mass[2] = fit_config_dict['model_parameter_means'][2] + print('True mass: ', fit_config_dict['model_parameter_means'][2]) + print('True all: ', fit_config_dict['model_parameter_means']) # get likleihood from fit - T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_true_mass) - fit_ll = T.PoissonLogLikelihood(results) + T.hist = T.TritiumSpectrumBackground(T.bin_centers, *fit_config_dict['model_parameter_means']) + fit_ll = -T.negPoissonLogLikelihood(results) # get likleihood of asimov best mass T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_best_mass) - best_ll = T.PoissonLogLikelihood(results) + best_ll = -T.negPoissonLogLikelihood(results) return results, T.hesse_errors, [fit_ll, best_ll] else: return results, T.hesse_errors, total_counts From ada791c8459d6a9d1e9e65119ca3cbc36367e4a5 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 10 Nov 2022 20:50:09 -0800 Subject: [PATCH 175/199] back to only using true mass --- mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 13f46131..4877d4f1 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -128,7 +128,7 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, print('True mass: ', fit_config_dict['model_parameter_means'][2]) print('True all: ', fit_config_dict['model_parameter_means']) # get likleihood from fit - T.hist = T.TritiumSpectrumBackground(T.bin_centers, *fit_config_dict['model_parameter_means']) + T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_true_mass) fit_ll = -T.negPoissonLogLikelihood(results) # get likleihood of asimov best mass T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_best_mass) From bbb7da5632b9dfc4c2011582fc493330e7c99990 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 10 Nov 2022 21:55:21 -0800 Subject: [PATCH 176/199] back to just using pos loglikelihood --- mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 4877d4f1..a38bbcf9 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -129,10 +129,10 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, print('True all: ', fit_config_dict['model_parameter_means']) # get likleihood from fit T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_true_mass) - fit_ll = -T.negPoissonLogLikelihood(results) + fit_ll = T.PoissonLogLikelihood(results) # get likleihood of asimov best mass T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_best_mass) - best_ll = -T.negPoissonLogLikelihood(results) + best_ll = T.PoissonLogLikelihood(results) return results, T.hesse_errors, [fit_ll, best_ll] else: return results, T.hesse_errors, total_counts From 6ff097e3fddac4a4e105db701baf379ab373c571 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 10 Nov 2022 23:26:22 -0800 Subject: [PATCH 177/199] added ll normalization --- mermithid/processors/Fitters/BinnedDataFitter.py | 4 +++- mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index c35fd74d..30ce671a 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -20,6 +20,7 @@ from morpho.utilities import morphologging, reader from morpho.processors import BaseProcessor from scipy.stats import chisquare +from scipy.special import factorial logger = morphologging.getLogger(__name__) @@ -290,7 +291,8 @@ def PoissonLogLikelihood(self, params): #print(self.bin_centers[index]) # poisson log likelihoood - ll = (self.hist[index]*np.log(expectation[index]) - expectation[index]).sum() + log_factorial = np.array([np.sum(np.log(np.arange(1, n+1))) for n in self.hist[index]]) + ll = (self.hist[index]*np.log(expectation[index]) - expectation[index]-log_factorial).sum() # extended ll: poisson total number of events N = np.nansum(expectation) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index a38bbcf9..a4a3801a 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -133,6 +133,7 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, # get likleihood of asimov best mass T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_best_mass) best_ll = T.PoissonLogLikelihood(results) + print(fit_ll, best_ll) return results, T.hesse_errors, [fit_ll, best_ll] else: return results, T.hesse_errors, total_counts From b5dde80164b85fb37900091b587c75f68aadf9c6 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Fri, 11 Nov 2022 08:55:16 -0800 Subject: [PATCH 178/199] comparing likelihoods of given all but mass parameter true --- .../processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index a4a3801a..7edb497f 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -121,9 +121,9 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, print(T.m_binned.covariance.correlation()) return results, T.minos_errors, total_counts elif fit_config_dict['return_ll']: - results_best_mass = deepcopy(results) + results_best_mass = deepcopy(fit_config_dict['model_parameter_means']) results_best_mass[2] = max(0, results[2]) - results_true_mass = deepcopy(results) + results_true_mass = deepcopy(fit_config_dict['model_parameter_means']) results_true_mass[2] = fit_config_dict['model_parameter_means'][2] print('True mass: ', fit_config_dict['model_parameter_means'][2]) print('True all: ', fit_config_dict['model_parameter_means']) @@ -133,7 +133,7 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, # get likleihood of asimov best mass T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_best_mass) best_ll = T.PoissonLogLikelihood(results) - print(fit_ll, best_ll) + return results, T.hesse_errors, [fit_ll, best_ll] else: return results, T.hesse_errors, total_counts From 2099fd50d5604c549702bf9b336d34da01fe69a4 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Sun, 13 Nov 2022 11:49:54 -0800 Subject: [PATCH 179/199] more normalization and using other parameters' best fit values again --- mermithid/processors/Fitters/BinnedDataFitter.py | 3 ++- .../processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mermithid/processors/Fitters/BinnedDataFitter.py b/mermithid/processors/Fitters/BinnedDataFitter.py index 30ce671a..285dd607 100644 --- a/mermithid/processors/Fitters/BinnedDataFitter.py +++ b/mermithid/processors/Fitters/BinnedDataFitter.py @@ -296,7 +296,8 @@ def PoissonLogLikelihood(self, params): # extended ll: poisson total number of events N = np.nansum(expectation) - extended_ll = -N+np.sum(self.hist)*np.log(N)+ll + log_factorial = np.sum(np.log(np.arange(1, np.sum(self.hist)+1))) + extended_ll = -N+np.sum(self.hist)*np.log(N)+ll-log_factorial return extended_ll diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 7edb497f..9777c68d 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -121,14 +121,14 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, print(T.m_binned.covariance.correlation()) return results, T.minos_errors, total_counts elif fit_config_dict['return_ll']: - results_best_mass = deepcopy(fit_config_dict['model_parameter_means']) + results_best_mass = deepcopy(results) results_best_mass[2] = max(0, results[2]) - results_true_mass = deepcopy(fit_config_dict['model_parameter_means']) + results_true_mass = deepcopy(results) results_true_mass[2] = fit_config_dict['model_parameter_means'][2] print('True mass: ', fit_config_dict['model_parameter_means'][2]) print('True all: ', fit_config_dict['model_parameter_means']) # get likleihood from fit - T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_true_mass) + #T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_true_mass) fit_ll = T.PoissonLogLikelihood(results) # get likleihood of asimov best mass T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_best_mass) From c45bf05ff3014267dfab5dfc94e4527beacb9a5b Mon Sep 17 00:00:00 2001 From: cclaessens Date: Sat, 19 Nov 2022 21:18:23 -0800 Subject: [PATCH 180/199] swapped expectation and observation for likelihood ratio --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 9777c68d..caaa3c3b 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -128,11 +128,11 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, print('True mass: ', fit_config_dict['model_parameter_means'][2]) print('True all: ', fit_config_dict['model_parameter_means']) # get likleihood from fit - #T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_true_mass) - fit_ll = T.PoissonLogLikelihood(results) + T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results) + fit_ll = T.PoissonLogLikelihood(results_true_mass) # get likleihood of asimov best mass - T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_best_mass) - best_ll = T.PoissonLogLikelihood(results) + #T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_best_mass) + best_ll = T.PoissonLogLikelihood(results_best_mass) return results, T.hesse_errors, [fit_ll, best_ll] else: @@ -1032,7 +1032,7 @@ def beta_rates(self, K, Q, m_nu_squared, shape='mainz'):#, index): Q_minus_K = Q-K if m_nu_squared >= 0: - nu_mass_shape_squared = (Q - K)**2 -m_nu_squared + nu_mass_shape_squared = (Q_minus_K)**2 -m_nu_squared index = np.where((nu_mass_shape_squared>0) & (Q_minus_K>0)) nu_mass_shape = np.sqrt(nu_mass_shape_squared[index]) spectrum[index] = (Q_minus_K[index])*nu_mass_shape#*self.ephasespace(K[index], Q) From 6282e2908649e71d3ec4603bef57f4de975f785a Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 23 Nov 2022 11:13:00 -0800 Subject: [PATCH 181/199] commented elif resolution is gaussian. also changed number of detector response stepsize back to old default. --- mermithid/misc/FakeTritiumDataFunctions.py | 4 ++-- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index f9aa3f02..9665ff70 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -339,11 +339,11 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, #convolved.append(convolved_j) convolved = np.concatenate(convolved_segments, axis=None) plt.savefig('varied_lineshapes.png', dpi=200) - elif resolution_function=='gaussian': + """elif resolution_function=='gaussian': lineshape_rates = np.flipud(lineshape_rates) beta_rates = spectral_rate(K, Q, mnu, final_state_array) convolved = convolve(beta_rates, lineshape_rates, mode='same') - np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) + np.put(convolved, below_Kmin, np.zeros(len(below_Kmin)))""" if (lineshape=='gaussian' or lineshape=='simplified_scattering' or lineshape=='simplified'): beta_rates = spectral_rate(K, Q, mnu, final_state_array) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 3bfe7c40..735adecb 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -230,7 +230,7 @@ def InternalConfigure(self, params): 'sigma_array': self.sigma_array, # This is an important parameter which determines how finely resolved the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to be increased for larger datasets. - 'num_points_in_std_array': 10000,#35846, + 'num_points_in_std_array': 35846, 'base_shape': 'dirac', 'path_to_osc_strengths_files': self.path_to_osc_strengths_files, 'path_to_scatter_spectra_file':self.path_to_scatter_spectra_file, From 09c8ad1c1f147d7810f4b09baa4db50e2c626afc Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 23 Nov 2022 16:56:26 -0800 Subject: [PATCH 182/199] adding frequency variation with gaussian resolution. Also adding effiicency function for sampling before interpolation. --- mermithid/misc/FakeTritiumDataFunctions.py | 33 ++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/mermithid/misc/FakeTritiumDataFunctions.py b/mermithid/misc/FakeTritiumDataFunctions.py index 59ac0913..9665ff70 100644 --- a/mermithid/misc/FakeTritiumDataFunctions.py +++ b/mermithid/misc/FakeTritiumDataFunctions.py @@ -282,8 +282,13 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, """K is an array-like object """ logger.info('Using scipy convolve') + logger.info('Lineshape is {} with {}'.format(lineshape, resolution_function)) energy_half_range = max(max_energy, abs(min_energy)) + #logger.info('Using {} frequency regions. Mean and std of p are {} and {}. For q its {} and {}'.format(len(ins_res_width_bounds)-1, + # np.mean(p_factors), np.std(p_factors), + # np.mean(q_factors), np.std(q_factors))) + if ins_res_width_bounds != None: Kbounds = [np.min(K)] + ins_res_width_bounds + [np.max(K)] else: @@ -312,35 +317,39 @@ def convolved_spectral_rate_arrays(K, Q, mnu, Kmin, lineshape_rates.append(np.flipud(complexLineShape.make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(scale_factors[i], ls_params[1], scatter_peak_ratio_p*p_factors[i], scatter_peak_ratio_q*q_factors[i], scatter_fraction, emitted_peak='dirac'))) elif resolution_function == 'gaussian_resolution' or resolution_function == 'gaussian': logger.warn("Scatter peak ratio function for lineshape with Gaussian resolution may not be up-to-date!") - lineshape_rates = complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(ls_params[0], ls_params[1], scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, emitted_peak='dirac') + gaussian_widths = [ls_params[0]*f for f in ins_res_width_factors] + lineshape_rates = [np.flipud(complexLineShape.make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(gaussian_widths[i], ls_params[1], scatter_peak_ratio_p*p_factors[i], scatter_peak_ratio_q*q_factors[i], scatter_fraction, emitted_peak='dirac')) for i in range(len(gaussian_widths))] else: logger.warn('{} is not a resolution function that has been implemented in the FakeDataGenerator'.format(resolution_function)) below_Kmin = np.where(K < Kmin) #Convolving - if (lineshape=='detailed_scattering' or lineshape=='detailed') and (resolution_function == 'simulated_resolution' or resolution_function == 'simulated'): + if (lineshape=='detailed_scattering' or lineshape=='detailed'):# and (resolution_function == 'simulated_resolution' or resolution_function == 'simulated'): convolved_segments = [] beta_rates = spectral_rate(K, Q, mnu, final_state_array) + plt.figure(figsize=(7,5)) for j in range(len(lineshape_rates)): #beta_rates = spectral_rate(K_segments[j], Q, mnu, final_state_array) + plt.plot(lineshape_rates[j]) convolved_j = convolve(beta_rates, lineshape_rates[j], mode='same') np.put(convolved_j, below_Kmin, np.zeros(len(below_Kmin))) #Only including the part of convolved_j that corresponds to the right values of K convolved_segments.append(convolved_j[np.logical_and(Kbounds[j]<=K, K<=Kbounds[j+1])]) #convolved.append(convolved_j) convolved = np.concatenate(convolved_segments, axis=None) - elif resolution_function=='gaussian': + plt.savefig('varied_lineshapes.png', dpi=200) + """elif resolution_function=='gaussian': lineshape_rates = np.flipud(lineshape_rates) beta_rates = spectral_rate(K, Q, mnu, final_state_array) convolved = convolve(beta_rates, lineshape_rates, mode='same') - np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) + np.put(convolved, below_Kmin, np.zeros(len(below_Kmin)))""" if (lineshape=='gaussian' or lineshape=='simplified_scattering' or lineshape=='simplified'): beta_rates = spectral_rate(K, Q, mnu, final_state_array) convolved = convolve(beta_rates, lineshape_rates, mode='same') np.put(convolved, below_Kmin, np.zeros(len(below_Kmin))) - + return convolved @@ -436,3 +445,17 @@ def efficiency_from_interpolation(x, efficiency_dict, B=0.9578186017836624): +def random_efficiency_from_interpolation(x, efficiency_dict, B=0.9578186017836624): + """ + Function to calculate efficiency + """ + logger.info('Sampling efficiencies before interpolation') + f = Frequency(x, B) + + efficiency_mean = efficiency_dict['eff interp with slope correction'] + efficiency_error = np.mean(efficiency_dict['error interp with slope correction'], axis=0) + random_efficiencies = np.random.normal(efficiency_mean, efficiency_error) + random_efficiencies[random_efficiencies<0] = 0. + interp_efficiency = interp1d(efficiency_dict['frequencies'], random_efficiencies, fill_value='0', bounds_error=False) + + return interp_efficiency(f) \ No newline at end of file From a91751e7021c8d4c70373a37306d59d335fdb219 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 23 Nov 2022 16:57:40 -0800 Subject: [PATCH 183/199] Sampling simulated resolution once during configuration. --- .../misc/MultiGasComplexLineShape.py | 256 ++++++++++-------- 1 file changed, 138 insertions(+), 118 deletions(-) diff --git a/mermithid/processors/misc/MultiGasComplexLineShape.py b/mermithid/processors/misc/MultiGasComplexLineShape.py index 062ea131..4f37a0d9 100644 --- a/mermithid/processors/misc/MultiGasComplexLineShape.py +++ b/mermithid/processors/misc/MultiGasComplexLineShape.py @@ -51,7 +51,7 @@ def InternalConfigure(self, params): # Read other parameters self.bins_choice = reader.read_param(params, 'bins_choice', []) self.gases = reader.read_param(params, 'gases', ["H2", "Kr", "He", "Ar"]) - # when self.fix_gas_composition and self.fix_width_scale_factor are both True, + # when self.fix_gas_composition and self.fix_width_scale_factor are both True, # fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor is used, # Then the first N-1 gas compositions are set below through self.scatter_fractions_for_gases # Otherwise, fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio is used @@ -117,7 +117,11 @@ def InternalConfigure(self, params): raise IOError('Path to osc strengths files does not exist') # Read shake parameters from JSON file if self.base_shape == 'shake': - self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) + self.shakeSpectrumClassInstance = ComplexLineShapeUtilities.ShakeSpectrumClass(self.shake_spectrum_parameters_json_path, self.std_eV_array()) + + # read in resolution if simulated + if 'simulated' in self.resolution_function: + self.sample_and_interpolate_resolution() return True def InternalRun(self): @@ -190,7 +194,7 @@ def std_lorenztian_17keV(self): x_array = self.std_eV_array() ans = lorentzian(x_array,0,kr_line_width) return ans - + #A Dirac delta functin def std_dirac(self): x_array = self.std_eV_array() @@ -204,7 +208,7 @@ def std_dirac(self): logger.warning('Lineshape will shift spectrum by > 1 eV') raise ValueError('problem with std_eV_array()') return ans - + # A gaussian function def gaussian(self, x_array, A, sigma, mu): f = A*(1./(sigma*np.sqrt(2*np.pi)))*np.exp(-(((x_array-mu)/sigma)**2.)/2.) @@ -278,7 +282,7 @@ def composite_gaussian_lorentzian(self, sigma): x_array = self.std_eV_array() w_g = x_array/sigma gamma = self.ratio_gamma_to_sigma*sigma - w_l = x_array/gamma + w_l = x_array/gamma lorentzian = 1./(gamma*np.pi)*1./(1+(w_l**2)) gaussian = 1./(np.sqrt(2.*np.pi)*sigma)*np.exp(-0.5*w_g**2) p = self.gaussian_proportion @@ -289,7 +293,7 @@ def elevated_gaussian(self, elevation_factor, sigma): x_array = self.std_eV_array() w_g = x_array/sigma gamma = self.ratio_gamma_to_sigma*sigma - w_l = x_array/gamma + w_l = x_array/gamma lorentzian = 1./(gamma*np.pi)*1./(1+(w_l**2)) gaussian = 1./(np.sqrt(2.*np.pi)*sigma)*np.exp(-0.5*w_g**2) modified_guassian_function = gaussian*(1 + elevation_factor*lorentzian) @@ -333,7 +337,7 @@ def another_scatter(self, input_spectrum, gas_type): f = signal.convolve(single,input_spectrum,mode='same') f_normed = self.normalize(f) return f_normed - + def radiation_loss_f(self): radiation_loss_data_file_path = self.path_to_missing_track_radiation_loss_data_numpy_file + '/missing_track_radiation_loss.npy' data_for_missing_track_radiation_loss = np.load(radiation_loss_data_file_path, allow_pickle = True) @@ -349,7 +353,7 @@ def radiation_loss_f(self): return f_radiation_energy_loss # Convolves the scatter functions and saves - # the results to a .npy file. + # the results to a .npy file. def generate_scatter_convolution_file(self): t = time.time() scatter_spectra_single_gas = {} @@ -387,7 +391,7 @@ def generate_scatter_convolution_file(self): mark_first_nonzero_component = 1 else: scatter_to_add = scatter_spectra_single_gas[gas_type][str(component).zfill(2)] - current_full_scatter = self.normalize(signal.convolve(current_full_scatter, scatter_to_add, mode='same')) + current_full_scatter = self.normalize(signal.convolve(current_full_scatter, scatter_to_add, mode='same')) scatter_spectra[entry_str] = current_full_scatter np.save(os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy'), scatter_spectra) elapsed = time.time() - t @@ -399,18 +403,18 @@ def generate_scatter_convolution_file(self): # If not, this function calls generate_scatter_convolution_file. # This function also checks to make sure that the scatter file have the correct # number of entries and correct number of points in the SELA, and if not, it generates a fresh file. - # When the variable regenerate is set as True, it generates a fresh file + # When the variable regenerate is set as True, it generates a fresh file def check_existence_of_scatter_file(self, regenerate = True): gases = self.gases if regenerate == True: logger.info('generate fresh scatter file') self.generate_scatter_convolution_file() - else: + else: directory = os.listdir(self.path_to_scatter_spectra_file) strippeddirs = [s.strip('\n') for s in directory] if 'scatter_spectra.npy' not in strippeddirs: self.generate_scatter_convolution_file() - test_file = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') + test_file = os.path.join(self.path_to_scatter_spectra_file, 'scatter_spectra.npy') test_dict = np.load(test_file, allow_pickle = True) N = len(self.gases) if len(test_dict.item()) != sum([comb(M + N -1, N -1) for M in range(1, self.max_scatters+1)]): @@ -419,7 +423,7 @@ def check_existence_of_scatter_file(self, regenerate = True): test_dict = np.load(test_file, allow_pickle = True) gas_str = gases[0] + '01' for gas in self.gases[1:]: - gas_str += gas + '00' + gas_str += gas + '00' if gas_str not in list(test_dict.item().keys()): print('Gas species not matching, generating fresh files') self.generate_scatter_convolution_files() @@ -432,7 +436,7 @@ def convolve_gaussian(self, func_to_convolve, gauss_FWHM_eV): ans = signal.convolve(resolution_f, func_to_convolve,mode='same') ans_normed = self.normalize(ans) return ans_normed - + def convolve_composite_gaussian(self, func_to_convolve, A_array, sigma_array): resolution_f = self.composite_gaussian(A_array, sigma_array) ans = signal.convolve(resolution_f, func_to_convolve, mode='same') @@ -494,7 +498,7 @@ def convolve_ins_resolution(self, working_spectrum): convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum - + def combine_four_trap_resolution_from_txt(self, trap_weights): if self.sample_ins_resolution_errors: weight_array = np.random.normal(trap_weights['weights'], trap_weights['errors']) @@ -509,7 +513,7 @@ def combine_four_trap_resolution_from_txt(self, trap_weights): y_data_combined = weight_array[0]*y_data_array[0] + weight_array[1]*y_data_array[1] + weight_array[2]*y_data_array[2] + weight_array[3]*y_data_array[3] y_err_data_combined = np.sqrt((weight_array[0]*y_err_data_array[0])**2 + (weight_array[1]*y_err_data_array[1])**2 + (weight_array[2]*y_err_data_array[2])**2 + (weight_array[3]*y_err_data_array[3])**2) return x_data, y_data_combined, y_err_data_combined - + def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_array): x_data, y_data_combined, y_err_data_combined = self.combine_four_trap_resolution_from_txt(weight_array) if self.sample_ins_resolution_errors: @@ -524,7 +528,7 @@ def convolve_ins_resolution_combining_four_trap(self, working_spectrum, weight_a return normalized_convolved_spectrum def convolve_simulated_resolution_scaled(self, working_spectrum, scale_factor): - if self.use_combined_four_trap_inst_reso: + """if self.use_combined_four_trap_inst_reso: x_data, y_data, y_err_data = self.combine_four_trap_resolution_from_txt(self.trap_weights) logger.info("Combined four instrumental resolution files") else: @@ -534,15 +538,31 @@ def convolve_simulated_resolution_scaled(self, working_spectrum, scale_factor): y_data = np.random.normal(y_data, y_err_data) logger.info("Sampling instrumental resolution counts per bin") scaled_xdata = x_data*scale_factor - f = interpolate.interp1d(x_data*scale_factor, y_data) + f = interpolate.interp1d(x_data*scale_factor, y_data)""" + x_array = self.std_eV_array() y_array = np.zeros(len(x_array)) - index_within_range_of_xdata = np.where((x_array >= scaled_xdata[0]) & (x_array <= scaled_xdata[-1])) - y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]) + + #index_within_range_of_xdata = np.where((x_array >= scaled_xdata[0]) & (x_array <= scaled_xdata[-1])) + #y_array[index_within_range_of_xdata] = f(x_array[index_within_range_of_xdata]/scale_factor) + y_array = self.interpolated_resolution(x_array/scale_factor) convolved_spectrum = signal.convolve(working_spectrum, y_array, mode = 'same') normalized_convolved_spectrum = self.normalize(convolved_spectrum) return normalized_convolved_spectrum + # do resolution sampling + def sample_and_interpolate_resolution(self): + if self.use_combined_four_trap_inst_reso: + x_data, y_data, y_err_data = self.combine_four_trap_resolution_from_txt(self.trap_weights) + logger.info("Combined four instrumental resolution files") + else: + x_data, y_data, y_err_data = self.read_ins_resolution_data(self.path_to_ins_resolution_data_txt) + logger.info("Using ONE simulated instrumental resolution file (not combining four)") + if self.sample_ins_resolution_errors: + y_data = np.random.normal(y_data, y_err_data) + logger.info("Sampling instrumental resolution counts per bin") + self.interpolated_resolution = interpolate.interp1d(x_data, y_data, fill_value=(0,0), bounds_error=False) + #might be untested def convolve_smeared_triangle(self, func_to_convolve, center, scale1, scale2, exponent, sigma): resolution_f = self.std_smeared_triangle(center, scale1, scale2, exponent, sigma) @@ -654,8 +674,8 @@ def chi_2_Poisson_simulated_resolution_scaled_fit_recon_eff(self, bin_centers, d def reduced_chi2_Pearson_Neyman_composite(self, data_hist_freq, fit_Hz, number_of_parameters): nonzero_bins_index = np.where(data_hist_freq != 0)[0] zero_bins_index = np.where(data_hist_freq == 0)[0] - fit_Hz_nonzero = fit_Hz[nonzero_bins_index] - data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] fit_Hz_zero = fit_Hz[zero_bins_index] data_Hz_zero = data_hist_freq[zero_bins_index] chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) @@ -722,7 +742,7 @@ def spectrum_func(self, bins_Hz, *p0): prob_parameter = p0[3] N = len(self.gases) scatter_proportion = p0[4:3+N] - + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) en_loss_array = self.std_eV_array() en_loss_array_min = en_loss_array[0] @@ -772,7 +792,7 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): scatter_proportion_max = 1 N = len(self.gases) p0_guess = [B_field_guess, FWHM_guess, amplitude_guess, prob_parameter_guess] + [scatter_proportion_guess]*(N-1) - p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min] + [scatter_proportion_min]*(N-1), + p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min] + [scatter_proportion_min]*(N-1), [B_field_max, FWHM_eV_max, amplitude_max, prob_parameter_max] + [scatter_proportion_max]*(N-1) ) # Actually do the fitting params , cov = curve_fit(self.spectrum_func, bins_Hz_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) @@ -793,16 +813,16 @@ def fit_data(self, freq_bins, data_hist_freq, print_params=True): prob_parameter_fit_err = perr[3] scatter_proportion_fit_err = list(perr[4:3+N])+[np.sqrt(sum(perr[4:3+N]**2))] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func(bins_Hz, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) - + nonzero_bins_index = np.where(data_hist_freq != 0)[0] zero_bins_index = np.where(data_hist_freq == 0)[0] - fit_Hz_nonzero = fit_Hz[nonzero_bins_index] - data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] fit_Hz_zero = fit_Hz[zero_bins_index] data_Hz_zero = data_hist_freq[zero_bins_index] chi2 = sum((fit_Hz_nonzero - data_Hz_nonzero)**2/data_Hz_nonzero) + sum((fit_Hz_nonzero - data_Hz_nonzero)**2/fit_Hz_nonzero) @@ -891,7 +911,7 @@ def spectrum_func_1(self, bins_Hz, *p0): FWHM_G_eV = p0[1] amplitude = p0[2] prob_parameter = p0[3] - + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) en_loss_array = self.std_eV_array() en_loss_array_min = en_loss_array[0] @@ -929,9 +949,9 @@ def fit_data_1(self, freq_bins, data_hist_freq): amplitude_max = np.sum(data_hist_freq)*3 prob_parameter_min = 1e-5 prob_parameter_max = 1 - - # p0_guess = [B_field_guess, FWHM_guess, amplitude_guess, prob_parameter_guess] - # p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min], + + # p0_guess = [B_field_guess, FWHM_guess, amplitude_guess, prob_parameter_guess] + # p0_bounds = ([B_field_min, FWHM_eV_min, amplitude_min, prob_parameter_min], # [B_field_max, FWHM_eV_max, amplitude_max, prob_parameter_max]) # Actually do the fitting # params , cov = curve_fit(self.spectrum_func_1, bins_Hz_nonzero, data_hist_nonzero, sigma=data_hist_err, p0=p0_guess, bounds=p0_bounds) @@ -959,7 +979,7 @@ def fit_data_1(self, freq_bins, data_hist_freq): amplitude_fit_err = perr[2] prob_parameter_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_1(bins_Hz, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 @@ -967,8 +987,8 @@ def fit_data_1(self, freq_bins, data_hist_freq): nonzero_bins_index = np.where(data_hist_freq != 0)[0] zero_bins_index = np.where(data_hist_freq == 0)[0] - fit_Hz_nonzero = fit_Hz[nonzero_bins_index] - data_Hz_nonzero = data_hist_freq[nonzero_bins_index] + fit_Hz_nonzero = fit_Hz[nonzero_bins_index] + data_Hz_nonzero = data_hist_freq[nonzero_bins_index] fit_Hz_zero = fit_Hz[zero_bins_index] data_Hz_zero = data_hist_freq[zero_bins_index] reduced_chi2 = self.reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = 4) @@ -1088,7 +1108,7 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): amplitude_max = np.sum(data_hist_freq)*3 prob_parameter_min = 1e-5 prob_parameter_max = 1 - + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess] p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max)] # Actually do the fitting @@ -1111,7 +1131,7 @@ def fit_data_ftc(self, freq_bins, data_hist_freq): amplitude_fit_err = perr[1] survival_prob_fit_err = perr[2] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_ftc(bins_Hz, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 @@ -1175,7 +1195,7 @@ def make_spectrum_ftc_2(self, prob_parameter, scatter_proportion, emitted_peak=' current_working_spectrum = self.convolve_ins_resolution_combining_four_trap(current_working_spectrum, self.trap_weights) else: current_working_spectrum = self.convolve_ins_resolution(current_working_spectrum) - + zeroth_order_peak = current_working_spectrum current_full_spectrum += current_working_spectrum N = len(self.gases) @@ -1265,7 +1285,7 @@ def fit_data_ftc_2(self, freq_bins, data_hist_freq): prob_parameter_fit_err = perr[2] scatter_proportion_fit_err = list(perr[3:2+N])+[np.sqrt(sum(perr[3:2+N]**2))] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_ftc_2(bins_Hz, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 @@ -1343,7 +1363,7 @@ def make_spectrum_smeared_triangle(self, prob_parameter, center, scale1, scale2, return current_full_spectrum def spectrum_func_smeared_triangle(self, bins_Hz, *p0): - + B_field = p0[0] amplitude = p0[1] prob_parameter = p0[2] @@ -1352,7 +1372,7 @@ def spectrum_func_smeared_triangle(self, bins_Hz, *p0): scale2 = p0[5] exponent = p0[6] sigma = p0[7] - + x_eV = ConversionFunctions.Energy(bins_Hz, B_field) en_loss_array = self.std_eV_array() en_loss_array_min = en_loss_array[0] @@ -1364,7 +1384,7 @@ def spectrum_func_smeared_triangle(self, bins_Hz, *p0): x_eV_minus_line = Constants.kr_line()*1000 - x_eV zero_idx = np.r_[np.where(x_eV_minus_line< en_loss_array_min)[0],np.where(x_eV_minus_line>en_loss_array_max)[0]] nonzero_idx = [i for i in range(len(x_eV)) if i not in zero_idx] - + full_spectrum = self.make_spectrum_smeared_triangle(prob_parameter, center, scale1, scale2, exponent, sigma) f_intermediate[nonzero_idx] = np.interp(x_eV_minus_line[nonzero_idx], en_loss_array, full_spectrum) f[nonzero_idx] += amplitude*f_intermediate[nonzero_idx]/np.sum(f_intermediate[nonzero_idx]) @@ -1397,9 +1417,9 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print width_max = ConversionFunctions.Energy(bins_Hz[0], B_field_guess) - ConversionFunctions.Energy(bins_Hz[-1], B_field_guess) exponent_min = 0.5 exponent_max = 2 - + p0_guess = [B_field_guess, amplitude_guess, prob_parameter_guess, center_guess, scale_guess, scale_guess, exponent_guess, sigma_guess] - p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (center_min, center_max), (width_min, width_max), (width_min, width_max), (exponent_min, exponent_max), (width_min, width_max)] + p0_bounds = [(B_field_min,B_field_max), (amplitude_min,amplitude_max), (prob_parameter_min, prob_parameter_max), (center_min, center_max), (width_min, width_max), (width_min, width_max), (exponent_min, exponent_max), (width_min, width_max)] #step_size = [1e-6, 5, 500, 0.1] # Actually do the fitting print(p0_guess) @@ -1432,12 +1452,12 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print exponent_fit_err = perr[6] sigma_fit_err = perr[7] total_counts_fit_err = amplitude_fit_err - + fit_Hz = spectrum_func(bins_Hz,*params) fit_keV = flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = flip_array(bins_keV) - reduced_chi2 = reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = len(params)) + reduced_chi2 = reduced_chi2_Poisson(data_hist_freq, fit_Hz, number_of_parameters = len(params)) if print_params == True: output_string = 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) output_string += '-----------------\n' @@ -1486,7 +1506,7 @@ def fit_data_smeared_triangle(self, RF_ROI_MIN, freq_bins, data_hist_freq, print 'reduced_chi2': reduced_chi2 } return dictionary_of_fit_results - + #fitting with commposite gaussian lorentzian resolution function and fixed scatter fraction def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion(self, survival_prob, sigma, emitted_peak='shake'): p = self.scatter_proportion @@ -1525,10 +1545,10 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion(self, s return current_full_spectrum def spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] - survival_prob = p0[2] + survival_prob = p0[2] sigma = p0[3] x_eV = ConversionFunctions.Energy(bins_Hz, B_field) @@ -1554,7 +1574,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion(self, freq_b self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -1606,13 +1626,13 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion(self, freq_b survival_prob_fit_err = perr[2] sigma_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -1686,9 +1706,9 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_scatter_proportion_and_sur return current_full_spectrum def spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] - amplitude = p0[1] + amplitude = p0[1] sigma = p0[2] x_eV = ConversionFunctions.Energy(bins_Hz, B_field) @@ -1714,7 +1734,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -1764,13 +1784,13 @@ def fit_data_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival amplitude_fit_err = perr[1] sigma_fit_err = perr[2] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_scatter_proportion_and_survival_prob(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -1842,9 +1862,9 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability(self, return current_full_spectrum def spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] - amplitude = p0[1] + amplitude = p0[1] sigma = p0[2] N = len(self.gases) scatter_proportion = p0[3: 2+N] @@ -1872,7 +1892,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability(self, freq self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -1924,13 +1944,13 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability(self, freq sigma_fit_err = perr[2] scatter_proportion_fit_err = list(perr[3:2+N]) + [np.sqrt(sum(perr[3:2+N]**2))] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2004,9 +2024,9 @@ def make_spectrum_composite_gaussian_lorentzian_fixed_survival_probability_parti return current_full_spectrum def spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] - amplitude = p0[1] + amplitude = p0[1] sigma = p0[2] N = len(self.free_gases) scatter_proportion = p0[3: 2+N] @@ -2034,7 +2054,7 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_ self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -2086,13 +2106,13 @@ def fit_data_composite_gaussian_lorentzian_fixed_survival_probability_partially_ sigma_fit_err = perr[2] scatter_proportion_fit_err = list(perr[3:2+N]) + [np.sqrt(sum(perr[3:2+N]**2))] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_lorentzian_fixed_survival_probability_partially_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2174,10 +2194,10 @@ def make_spectrum_elevated_gaussian_fixed_scatter_proportion(self, survival_prob return current_full_spectrum def spectrum_func_elevated_gaussian_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] - survival_prob = p0[2] + survival_prob = p0[2] sigma = p0[3] elevation_factor = p0[4] @@ -2204,7 +2224,7 @@ def fit_data_elevated_gaussian_fixed_scatter_proportion(self, freq_bins, data_hi self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -2261,13 +2281,13 @@ def fit_data_elevated_gaussian_fixed_scatter_proportion(self, freq_bins, data_hi sigma_fit_err = perr[3] elevation_factor_fit_err = perr[4] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_elevated_gaussian_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2342,7 +2362,7 @@ def make_spectrum_composite_gaussian_fixed_scatter_proportion(self, survival_pro return current_full_spectrum def spectrum_func_composite_gaussian_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] survival_prob = p0[2] @@ -2370,7 +2390,7 @@ def fit_data_composite_gaussian_fixed_scatter_proportion(self, freq_bins, data_h self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -2423,13 +2443,13 @@ def fit_data_composite_gaussian_fixed_scatter_proportion(self, freq_bins, data_h amplitude_fit_err = perr[1] survival_prob_fit_err = perr[2] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2498,7 +2518,7 @@ def make_spectrum_composite_gaussian_pedestal_factor_fixed_scatter_proportion(se return current_full_spectrum def spectrum_func_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] survival_prob = p0[2] @@ -2527,7 +2547,7 @@ def fit_data_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, f self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -2582,13 +2602,13 @@ def fit_data_composite_gaussian_pedestal_factor_fixed_scatter_proportion(self, f survival_prob_fit_err = perr[2] pedestal_factor_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_pedestal_factor_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2658,7 +2678,7 @@ def make_spectrum_composite_gaussian_scaled_fixed_scatter_proportion(self, survi return current_full_spectrum def spectrum_func_composite_gaussian_scaled_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] survival_prob = p0[2] @@ -2687,7 +2707,7 @@ def fit_data_composite_gaussian_scaled_fixed_scatter_proportion(self, freq_bins, self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -2742,13 +2762,13 @@ def fit_data_composite_gaussian_scaled_fixed_scatter_proportion(self, freq_bins, survival_prob_fit_err = perr[2] scale_factor_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_composite_gaussian_scaled_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2819,7 +2839,7 @@ def make_spectrum_simulated_resolution_scaled_fixed_scatter_proportion(self, sur return current_full_spectrum def spectrum_func_simulated_resolution_scaled_fixed_scatter_proportion(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] survival_prob = p0[2] @@ -2848,7 +2868,7 @@ def fit_data_simulated_resolution_scaled_fixed_scatter_proportion(self, freq_bin self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -2903,13 +2923,13 @@ def fit_data_simulated_resolution_scaled_fixed_scatter_proportion(self, freq_bin survival_prob_fit_err = perr[2] scale_factor_fit_err = perr[3] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fixed_scatter_proportion(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -2976,7 +2996,7 @@ def make_spectrum_simulated_resolution_scaled_fit_recon_eff(self, survival_prob, return current_full_spectrum def spectrum_func_simulated_resolution_scaled_fit_recon_eff(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] survival_prob = p0[2] @@ -3008,7 +3028,7 @@ def fit_data_simulated_resolution_scaled_fit_recon_eff(self, freq_bins, data_his self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - + quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -3071,13 +3091,13 @@ def fit_data_simulated_resolution_scaled_fit_recon_eff(self, freq_bins, data_his recon_eff_b_fit_err = perr[5] recon_eff_c_fit_err = perr[6] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_recon_eff(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -3151,7 +3171,7 @@ def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio(self, scale return current_full_spectrum def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] scale_factor = p0[2] @@ -3192,8 +3212,8 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, t = time.time() self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN - bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - if self.use_quad_trap_eff_interp == True: + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + if self.use_quad_trap_eff_interp == True: quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -3221,7 +3241,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, survival_probability_min = 1e-5 survival_probability_max = 1 scatter_fraction_min = 1e-5 - scatter_fraction_max = 1 + scatter_fraction_max = 1 scale_factor_min = 1e-5 scale_factor_max = 5 scatter_peak_ratio_parameter_min = 1e-5 @@ -3253,7 +3273,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, scatter_peak_ratio_q_fit = params[5] total_counts_fit = amplitude_fit logger.info('\n'+str(m_binned.params)) - scatter_fraction_fit = params[6:5+N]+[1- sum(params[6:5+N])] + scatter_fraction_fit = params[6:5+N]+[1- sum(params[6:5+N])] perr = m_binned.errors[0:] B_field_fit_err = perr[0] @@ -3264,14 +3284,14 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, scatter_peak_ratio_q_fit_err = perr[5] total_counts_fit_err = amplitude_fit_err scatter_fraction_fit_err = perr[6:5+N]+[np.sqrt(sum(np.array(perr[6:5+N])**2))] - + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) correlation_matrix = m_binned.covariance.correlation() - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -3318,7 +3338,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio(self, freq_bins, return dictionary_of_fit_results - def make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(self, gauss_FWHM_eV, survival_probability, scatter_peak_ratio_b, scatter_peak_ratio_c, scatter_fraction, emitted_peak='shake'): + def make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(self, gauss_FWHM_eV, survival_probability, scatter_peak_ratio_p, scatter_peak_ratio_q, scatter_fraction, emitted_peak='shake'): p = np.zeros(len(self.gases)) p[0:-1] = scatter_fraction p[-1] = 1 - sum(scatter_fraction) @@ -3337,7 +3357,7 @@ def make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(self, gauss_FWHM_eV current_full_spectrum += zeroth_order_peak N = len(self.gases) for M in range(1, self.max_scatters + 1): - scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_b*M**scatter_peak_ratio_c) + scatter_peak_ratio = np.exp(-1.*scatter_peak_ratio_p*M**( -self.factor*scatter_peak_ratio_p + scatter_peak_ratio_q))#np.exp(-1.*scatter_peak_ratio_b*M**scatter_peak_ratio_c) gas_scatter_combinations = np.array([np.array(i) for i in product(range(M+1), repeat=N) if sum(i)==M]) for combination in gas_scatter_combinations: #print(combination) @@ -3354,7 +3374,7 @@ def make_spectrum_gaussian_resolution_fit_scatter_peak_ratio(self, gauss_FWHM_eV return current_full_spectrum def spectrum_func_gaussian_resolution_fit_scatter_peak_ratio(self, bins_Hz, eff_array, *p0): - + B_field = p0[0] amplitude = p0[1] gauss_FWHM_eV = p0[2] @@ -3396,7 +3416,7 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - if self.use_quad_trap_eff_interp == True: + if self.use_quad_trap_eff_interp == True: quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -3420,7 +3440,7 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi survival_probability_min = 1e-5 survival_probability_max = 1 scatter_fraction_min = 1e-5 - scatter_fraction_max = 1 + scatter_fraction_max = 1 scale_factor_min = 1e-5 scale_factor_max = 5 scatter_peak_ratio_parameter_min = 1e-5 @@ -3452,7 +3472,7 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi scatter_peak_ratio_c_fit = params[5] total_counts_fit = amplitude_fit logger.info('\n'+str(m_binned.params)) - scatter_fraction_fit = params[6:5+N]+[1- sum(params[6:5+N])] + scatter_fraction_fit = params[6:5+N]+[1- sum(params[6:5+N])] perr = m_binned.errors[0:] B_field_fit_err = perr[0] @@ -3463,13 +3483,13 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi scatter_peak_ratio_c_fit_err = perr[5] total_counts_fit_err = amplitude_fit_err scatter_fraction_fit_err = perr[6:5+N]+[np.sqrt(sum(np.array(perr[6:5+N])**2))] - + fit_Hz = self.spectrum_func_gaussian_resolution_fit_scatter_peak_ratio(bins_Hz, eff_array, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) @@ -3512,11 +3532,11 @@ def fit_data_gaussian_resolution_fit_scatter_peak_ratio(self, freq_bins, data_hi 'data_hist_freq': data_hist_freq, 'reduced_chi2': reduced_chi2 } - + return dictionary_of_fit_results def generate_scatter_peaks(self): - + p = np.zeros(len(self.gases)) scatter_fraction = self.scatter_fractions_for_gases p[0:-1] = scatter_fraction @@ -3534,7 +3554,7 @@ def generate_scatter_peaks(self): current_working_spectrum = self.shakeSpectrumClassInstance.shake_spectrum() elif emitted_peak == 'dirac': current_working_spectrum = self.std_dirac() - + scale_factor = 1 current_working_spectrum = self.convolve_simulated_resolution_scaled(current_working_spectrum, scale_factor) zeroth_order_peak = current_working_spectrum @@ -3570,7 +3590,7 @@ def make_spectrum_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_ return current_full_spectrum def spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(self, bins_Hz, eff_array, scatter_peaks, *p0): - + B_field = p0[0] amplitude = p0[1] survival_probability = p0[2] @@ -3608,8 +3628,8 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c t = time.time() self.check_existence_of_scatter_file() bins_Hz = freq_bins + self.RF_ROI_MIN - bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) - if self.use_quad_trap_eff_interp == True: + bins_Hz = 0.5*(bins_Hz[1:] + bins_Hz[:-1]) + if self.use_quad_trap_eff_interp == True: quad_trap_interp = np.load(self.path_to_quad_trap_eff_interp, allow_pickle = True) quad_trap_count_rate_interp = quad_trap_interp.item()['count_rate_interp'] eff_array = quad_trap_count_rate_interp(bins_Hz) @@ -3636,7 +3656,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c survival_probability_min = 1e-5 survival_probability_max = 1 scatter_fraction_min = 1e-5 - scatter_fraction_max = 1 + scatter_fraction_max = 1 scale_factor_min = 1e-5 scale_factor_max = 5 scatter_peak_ratio_parameter_min = 1e-5 @@ -3648,7 +3668,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c p0_guess = [B_field_guess, amplitude_guess, survival_probability_guess, scatter_peak_ratio_parameter_guess, scatter_peak_ratio_parameter_guess] p0_bounds = [(B_field_min,B_field_max), (amplitude_min, amplitude_max), (survival_probability_min, survival_probability_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max), (scatter_peak_ratio_parameter_min, scatter_peak_ratio_parameter_max)] parameter_names = ['B field','amplitude', 'survival probability','scatter peak ratio param p', 'scatter peak ratio param q'] - + scatter_peaks = self.generate_scatter_peaks() # Actually do the fitting m_binned = Minuit(lambda p: self.chi_2_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(bins_Hz, data_hist_freq, eff_array, scatter_peaks, p), p0_guess, name = parameter_names) @@ -3668,7 +3688,7 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c scatter_peak_ratio_p_fit = params[3] scatter_peak_ratio_q_fit = params[4] total_counts_fit = amplitude_fit - logger.info('\n'+str(m_binned.params)) + logger.info('\n'+str(m_binned.params)) perr = m_binned.errors[0:] B_field_fit_err = perr[0] @@ -3677,14 +3697,14 @@ def fit_data_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_c scatter_peak_ratio_p_fit_err = perr[3] scatter_peak_ratio_q_fit_err = perr[4] total_counts_fit_err = amplitude_fit_err - + fit_Hz = self.spectrum_func_simulated_resolution_scaled_fit_scatter_peak_ratio_with_fixed_gas_composition_and_width_scale_factor(bins_Hz, eff_array, scatter_peaks, *params) fit_keV = ComplexLineShapeUtilities.flip_array(fit_Hz) bins_keV = ConversionFunctions.Energy(bins_Hz, B_field_fit)/1000 bins_keV = ComplexLineShapeUtilities.flip_array(bins_keV) reduced_chi2 = m_binned.fval/(len(fit_Hz)-m_binned.nfit) correlation_matrix = m_binned.covariance.correlation() - + if print_params == True: output_string = '\n' output_string += 'Reduced chi^2 = {:.2e}\n'.format(reduced_chi2) From de870bdad4b38036a8b88b1d25ad76704ed3b810 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 23 Nov 2022 16:58:58 -0800 Subject: [PATCH 184/199] Response energy stepzise can be modified. Efficiency is sampled first, then interpolated. --- .../TritiumSpectrum/FakeDataGenerator.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 138f9ec2..735adecb 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -230,7 +230,7 @@ def InternalConfigure(self, params): 'sigma_array': self.sigma_array, # This is an important parameter which determines how finely resolved the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to be increased for larger datasets. - 'num_points_in_std_array':35846, + 'num_points_in_std_array': 35846, 'base_shape': 'dirac', 'path_to_osc_strengths_files': self.path_to_osc_strengths_files, 'path_to_scatter_spectra_file':self.path_to_scatter_spectra_file, @@ -246,6 +246,8 @@ def InternalConfigure(self, params): self.complexLineShape.Configure(complexLineShape_config) logger.info('Checking existence of scatter spectra files') self.complexLineShape.check_existence_of_scatter_file() + lineshape_array = self.complexLineShape.std_eV_array() + self.lineshape_stepize = lineshape_array[1]-lineshape_array[0] else: logger.error("'detailed_or_simplified' is neither 'detailed' nor 'simplified'") return False @@ -373,13 +375,15 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 logger.info('Stepsize is {} eV'.format(step_size)) #Options of kinetic energies to be sampled - self.Koptions = np.arange(Kmin_eff, Kmax_eff, step_size) + self.Koptions = np.arange(Kmin_eff, Kmax_eff, self.lineshape_stepize) if efficiency_dict is not None: logger.info('Evaluating efficiencies') - efficiency_mean, efficiency_error = efficiency_from_interpolation(self.Koptions, efficiency_dict, B_field) + """efficiency_mean, efficiency_error = efficiency_from_interpolation(self.Koptions, efficiency_dict, B_field) logger.info("Sampling efficiencies given means and uncertainties") - efficiency = np.random.normal(efficiency_mean, efficiency_error) + efficiency = np.random.normal(efficiency_mean, efficiency_error)""" + + efficiency = random_efficiency_from_interpolation(self.Koptions, efficiency_dict, B_field) eff_negative = (efficiency<0.) efficiency[eff_negative] = 0. #Whenever this occurs, efficiency_mean=0 and efficiency_error=1 else: @@ -431,7 +435,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 ratesB = convolve(ratesB, gaussian_rates, mode='same') #if self.ins_res_width_bounds==None: ratesS = convolve(ratesS, gaussian_rates, mode='same') - + ratesS[ratesS<0.] = 0. rate_sumS = np.sum(ratesS) probsS = np.array(ratesS)/rate_sumS @@ -443,7 +447,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 rate_sumS.append(np.sum(ratesS[i])) probsS.append(np.array(ratesS[i])/rate_sumS[i]) """ - + ratesB[ratesB<0.] = 0. rate_sumB = np.sum(ratesB) probsB = np.array(ratesB)/rate_sumB @@ -460,6 +464,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 #if self.ins_res_width_bounds==None: temp_Koptions, temp_probsS, temp_probsB = self.Koptions, probsS, probsB split_Koptions, split_probsS, split_probsB = [], [], [] + print(len(temp_Koptions), len(temp_probsS)) for i in range(len(self.channel_bounds)): split_Koptions.append(temp_Koptions[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) split_probsS.append(temp_probsS[Frequency(temp_Koptions, self.B_field)<=self.channel_bounds[i]]) @@ -474,7 +479,7 @@ def generate_unbinned_data(self, Q_mean, mass, ROIbound, S, B_1kev, nsteps=10**4 rates = [] for i in range(len(self.channel_runtimes)): - rates.append((S*runtime_ratios[i]*split_probsS[i] + B*split_probsB[i])/(S*runtime_ratios[i]+B)) + rates.append((S*runtime_ratios[i]*split_probsS[i] + B*split_probsB[i])/(S*runtime_ratios[i]+B)) self.Koptions = np.concatenate(split_Koptions) rates = np.concatenate(rates) From f600baf203e97c4cd2565a1205d82e2b1f4c10e2 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Sat, 3 Dec 2022 10:51:55 -0800 Subject: [PATCH 185/199] added correlation from max SNR of p q with width resolution in frequentist analysis. --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index caaa3c3b..81fe3d36 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -191,7 +191,7 @@ def InternalConfigure(self, config_dict): self.helium_lineshape_path = reader.read_param(config_dict, 'helium_lineshape_path', "optional") self.use_helium_scattering = reader.read_param(config_dict, 'use_helium_scattering', False) self.use_frequency_dependent_lineshape = reader.read_param(config_dict, 'use_frequency_dependent_lineshape', True) - self.correlated_p_q_scale = reader.read_param(config_dict, 'correlated_p_q_scale', False) + self.correlated_p_q_res = reader.read_param(config_dict, 'correlated_p_q_res', False) self.correlated_p_q = reader.read_param(config_dict, 'correlated_p_q', False) self.derived_two_gaussian_model = reader.read_param(config_dict, 'derived_two_gaussian_model', True) @@ -313,12 +313,14 @@ def InternalConfigure(self, config_dict): self.p_q_corr = reader.read_param(config_dict, 'p_q_corr', 0) self.p_scale_corr = reader.read_param(config_dict, 'p_scale_corr', 0) self.q_scale_corr = reader.read_param(config_dict, 'q_scale_corr', 0) + self.p_q_max_snr_stds = reader.read_param(config_dict, 'p_q_max_snr_stds', [0, 0]) - stds = [self.scatter_peak_ratio_p_width, self.scatter_peak_ratio_q_width, self.scale_width] - self.p_q_scale_cov_matrix = [[stds[0]**2, self.p_q_corr*stds[0]*stds[1], self.p_scale_corr*stds[0]*stds[2]], - [self.p_q_corr*stds[1]*stds[0], stds[1]**2, self.q_scale_corr*stds[1]*stds[2]], - [self.p_scale_corr*stds[2]*stds[0], self.q_scale_corr*stds[2]*stds[1], stds[2]**2]] + stds = [self.scatter_peak_ratio_p_width, self.scatter_peak_ratio_q_width, self.res_width] + self.p_q_res_cov_matrix = [[stds[0]**2, self.p_q_corr*stds[0]*stds[1], self.p_scale_corr*self.p_q_max_snr_stds[0]*self.scale_width*self.res_mean], + [self.p_q_corr*stds[0]*stds[1], stds[1]**2, self.q_scale_corr*self.p_q_max_snr_stds[1]*self.scale_width*self.res_mean], + [self.p_scale_corr*self.p_q_max_snr_stds[0]*self.scale_width*self.res_mean, self.q_scale_corr*self.p_q_max_snr_stds[1]*self.scale_width*self.res_mean, stds[2]**2]] + print(self.p_q_max_snr_stds, self.p_scale_corr, self.scale_width*self.res_mean ) self.p_q_cov_matrix = [[stds[0]**2, self.p_q_corr*stds[0]*stds[1]], [self.p_q_corr*stds[1]*stds[0], stds[1]**2]] @@ -432,9 +434,15 @@ def InternalConfigure(self, config_dict): if i in self.constrained_parameters: self.fixed_parameters[i] = False - if self.is_scattered and self.correlated_p_q: + print('\tHere comes the covariance matrix') + if self.is_scattered and self.correlated_p_q_res: + self.correlated_parameters = [self.scatter_peak_ratio_p_index, self.scatter_peak_ratio_q_index, self.res_index] + self.cov_matrix = self.p_q_res_cov_matrix + print(self.cov_matrix) + elif self.is_scattered and self.correlated_p_q: self.correlated_parameters = [self.scatter_peak_ratio_p_index, self.scatter_peak_ratio_q_index] self.cov_matrix = self.p_q_cov_matrix + print(self.cov_matrix) else: self.correlated_parameters = [] From 657a56ebe71aa5bcc9ea513b4a96c2fe8aa066b4 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Sat, 3 Dec 2022 11:09:33 -0800 Subject: [PATCH 186/199] using actual max SNR uncertainty on sigma --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 81fe3d36..2a22fc67 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -253,6 +253,7 @@ def InternalConfigure(self, config_dict): self.res_mean = reader.read_param(config_dict, 'gaussian_resolution_mean', 15.0) self.res_width = reader.read_param(config_dict, 'gaussian_resolution_width', 1.0) + self.res_width_from_maxSNR = reader.read_param(config_dict, 'sigma_std_maxSNR', 0) self.res = self.res_mean if self.resolution_model != 'gaussian': @@ -317,10 +318,9 @@ def InternalConfigure(self, config_dict): stds = [self.scatter_peak_ratio_p_width, self.scatter_peak_ratio_q_width, self.res_width] - self.p_q_res_cov_matrix = [[stds[0]**2, self.p_q_corr*stds[0]*stds[1], self.p_scale_corr*self.p_q_max_snr_stds[0]*self.scale_width*self.res_mean], - [self.p_q_corr*stds[0]*stds[1], stds[1]**2, self.q_scale_corr*self.p_q_max_snr_stds[1]*self.scale_width*self.res_mean], - [self.p_scale_corr*self.p_q_max_snr_stds[0]*self.scale_width*self.res_mean, self.q_scale_corr*self.p_q_max_snr_stds[1]*self.scale_width*self.res_mean, stds[2]**2]] - print(self.p_q_max_snr_stds, self.p_scale_corr, self.scale_width*self.res_mean ) + self.p_q_res_cov_matrix = [[stds[0]**2, self.p_q_corr*stds[0]*stds[1], self.p_scale_corr*self.p_q_max_snr_stds[0]*self.res_width_from_maxSNR], + [self.p_q_corr*stds[0]*stds[1], stds[1]**2, self.q_scale_corr*self.p_q_max_snr_stds[1]*self.res_width_from_maxSNR], + [self.p_scale_corr*self.p_q_max_snr_stds[0]*self.res_width_from_maxSNR, self.q_scale_corr*self.p_q_max_snr_stds[1]*self.res_width_from_maxSNR, stds[2]**2]] self.p_q_cov_matrix = [[stds[0]**2, self.p_q_corr*stds[0]*stds[1]], [self.p_q_corr*stds[1]*stds[0], stds[1]**2]] @@ -434,15 +434,15 @@ def InternalConfigure(self, config_dict): if i in self.constrained_parameters: self.fixed_parameters[i] = False - print('\tHere comes the covariance matrix') + logger.info('Here comes the covariance matrix') if self.is_scattered and self.correlated_p_q_res: self.correlated_parameters = [self.scatter_peak_ratio_p_index, self.scatter_peak_ratio_q_index, self.res_index] self.cov_matrix = self.p_q_res_cov_matrix - print(self.cov_matrix) + logger.info(self.cov_matrix) elif self.is_scattered and self.correlated_p_q: self.correlated_parameters = [self.scatter_peak_ratio_p_index, self.scatter_peak_ratio_q_index] self.cov_matrix = self.p_q_cov_matrix - print(self.cov_matrix) + logger.info(self.cov_matrix) else: self.correlated_parameters = [] From 4aa16cd3a3406886ecdc596985134dc23464161e Mon Sep 17 00:00:00 2001 From: cclaessens Date: Mon, 5 Dec 2022 08:43:08 -0800 Subject: [PATCH 187/199] preparing for FC jobs --- mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 2 +- mermithid/processors/TritiumSpectrum/FakeDataGenerator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 2a22fc67..12f2b18d 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -120,7 +120,7 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, if fit_config_dict['minos_intervals']: print(T.m_binned.covariance.correlation()) return results, T.minos_errors, total_counts - elif fit_config_dict['return_ll']: + elif 'return_ll' in fit_config_dict and fit_config_dict['return_ll']: results_best_mass = deepcopy(results) results_best_mass[2] = max(0, results[2]) results_true_mass = deepcopy(results) diff --git a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py index 735adecb..3bfe7c40 100644 --- a/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py +++ b/mermithid/processors/TritiumSpectrum/FakeDataGenerator.py @@ -230,7 +230,7 @@ def InternalConfigure(self, params): 'sigma_array': self.sigma_array, # This is an important parameter which determines how finely resolved the scatter calculations are. 10000 seems to produce a stable fit with minimal slowdown, for ~4000 fake events. The parameter may need to be increased for larger datasets. - 'num_points_in_std_array': 35846, + 'num_points_in_std_array': 10000,#35846, 'base_shape': 'dirac', 'path_to_osc_strengths_files': self.path_to_osc_strengths_files, 'path_to_scatter_spectra_file':self.path_to_scatter_spectra_file, From f450296f33ecfb336742d396e8ad801faa16a22b Mon Sep 17 00:00:00 2001 From: cclaessens Date: Tue, 6 Dec 2022 14:19:14 -0800 Subject: [PATCH 188/199] preparing for llnl FC jobs --- .../processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 12f2b18d..a9c274ba 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -434,15 +434,15 @@ def InternalConfigure(self, config_dict): if i in self.constrained_parameters: self.fixed_parameters[i] = False - logger.info('Here comes the covariance matrix') + #logger.info('Here comes the covariance matrix') if self.is_scattered and self.correlated_p_q_res: self.correlated_parameters = [self.scatter_peak_ratio_p_index, self.scatter_peak_ratio_q_index, self.res_index] self.cov_matrix = self.p_q_res_cov_matrix - logger.info(self.cov_matrix) + #logger.info(self.cov_matrix) elif self.is_scattered and self.correlated_p_q: self.correlated_parameters = [self.scatter_peak_ratio_p_index, self.scatter_peak_ratio_q_index] self.cov_matrix = self.p_q_cov_matrix - logger.info(self.cov_matrix) + #logger.info(self.cov_matrix) else: self.correlated_parameters = [] @@ -1034,7 +1034,7 @@ def derived_two_gaussian_resolution(self, energy_array, sigma_s, mu_1, mu_2, A=1 return lineshape - def beta_rates(self, K, Q, m_nu_squared, shape='mainz'):#, index): + def beta_rates(self, K, Q, m_nu_squared, shape='llnl'):#, index): spectrum = np.zeros(len(K)) Q_minus_K = Q-K From 6493d58a24049af652f828b1f8aa1d44bc6368e9 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Thu, 8 Dec 2022 13:36:29 -0800 Subject: [PATCH 189/199] =?UTF-8?q?making=20-mbeta=C2=B2=20model=20configu?= =?UTF-8?q?rable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index a9c274ba..296e1903 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -179,6 +179,10 @@ def InternalConfigure(self, config_dict): self.fit_efficiency_tilt = reader.read_param(config_dict, 'fit_efficiency_tilt', False) # efficiency slope is free parameter self.fit_nu_mass = reader.read_param(config_dict, 'fit_neutrino_mass', False) self.use_relative_livetime_correction = reader.read_param(config_dict, 'use_relative_livetime_correction', False) + self.neg_mbeta_squared_model = reader.read_param(config_dict, 'neg_mbeta_squared_model', 'katrin') + + if self.fit_nu_mass: + logger.info('Using {} model'.format(self.neg_mbeta_squared_model)) # save plots in (processor can plot lineshape used in tritium model) self.savepath = reader.read_param(config_dict, 'savepath', '.') @@ -825,6 +829,7 @@ def SamplePriors(self, sampled_parameters): self.B = self.Gaussian_sample(self.B_mean, self.B_width) self.parameter_samples['B'] = self.B sample_values.append(self.B) + logger.info('B field prior: {} +/- {}'.format(self.B_mean, self.B_width)) if 'endpoint' in sampled_parameters.keys() and sampled_parameters['endpoint']: self.model_parameter_means[self.endpoint_index] = self.Gaussian_sample(self.endpoint_mean, self.endpoint_width) self.parameter_samples['endpoint'] = self.endpoint @@ -1034,8 +1039,9 @@ def derived_two_gaussian_resolution(self, energy_array, sigma_s, mu_1, mu_2, A=1 return lineshape - def beta_rates(self, K, Q, m_nu_squared, shape='llnl'):#, index): + def beta_rates(self, K, Q, m_nu_squared, shape='mainz'):#, index): spectrum = np.zeros(len(K)) + shape = self.neg_mbeta_squared_model Q_minus_K = Q-K From 1d296d7baf1277cd4de2670ff74f0bc5b6304e24 Mon Sep 17 00:00:00 2001 From: cclaessens Date: Tue, 17 Jan 2023 13:04:11 -0800 Subject: [PATCH 190/199] trying mean priors instead of best fits for other parameters to generage FC plot --- .../TritiumSpectrum/BinnedTritiumMLFitter.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 296e1903..4f508165 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -121,9 +121,13 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, print(T.m_binned.covariance.correlation()) return results, T.minos_errors, total_counts elif 'return_ll' in fit_config_dict and fit_config_dict['return_ll']: - results_best_mass = deepcopy(results) + + #all_params = results + all_params = fit_config_dict['model_parameter_means'] + + results_best_mass = deepcopy(all_params) results_best_mass[2] = max(0, results[2]) - results_true_mass = deepcopy(results) + results_true_mass = deepcopy(all_params) results_true_mass[2] = fit_config_dict['model_parameter_means'][2] print('True mass: ', fit_config_dict['model_parameter_means'][2]) print('True all: ', fit_config_dict['model_parameter_means']) @@ -179,7 +183,7 @@ def InternalConfigure(self, config_dict): self.fit_efficiency_tilt = reader.read_param(config_dict, 'fit_efficiency_tilt', False) # efficiency slope is free parameter self.fit_nu_mass = reader.read_param(config_dict, 'fit_neutrino_mass', False) self.use_relative_livetime_correction = reader.read_param(config_dict, 'use_relative_livetime_correction', False) - self.neg_mbeta_squared_model = reader.read_param(config_dict, 'neg_mbeta_squared_model', 'katrin') + self.neg_mbeta_squared_model = reader.read_param(config_dict, 'neg_mbeta_squared_model', 'mainz') if self.fit_nu_mass: logger.info('Using {} model'.format(self.neg_mbeta_squared_model)) From 7de432728fba38a76b0ccd73647951037668b1de Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 18 Jan 2023 09:55:55 -0800 Subject: [PATCH 191/199] going back to using best fit values --- mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 4f508165..e50f0302 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -122,8 +122,8 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, return results, T.minos_errors, total_counts elif 'return_ll' in fit_config_dict and fit_config_dict['return_ll']: - #all_params = results - all_params = fit_config_dict['model_parameter_means'] + all_params = results + #all_params = fit_config_dict['model_parameter_means'] results_best_mass = deepcopy(all_params) results_best_mass[2] = max(0, results[2]) From da47a46a71140e421b5444eff1c2f959bfa7cfdd Mon Sep 17 00:00:00 2001 From: cclaessens Date: Wed, 18 Jan 2023 10:06:55 -0800 Subject: [PATCH 192/199] setting background to zero --- .../processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index e50f0302..93524e64 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -122,9 +122,13 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, return results, T.minos_errors, total_counts elif 'return_ll' in fit_config_dict and fit_config_dict['return_ll']: - all_params = results + modified_results = deepcopy(results) + modified_results[1] = 0 + all_params = modified_results #all_params = fit_config_dict['model_parameter_means'] + + results_best_mass = deepcopy(all_params) results_best_mass[2] = max(0, results[2]) results_true_mass = deepcopy(all_params) @@ -132,7 +136,7 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, print('True mass: ', fit_config_dict['model_parameter_means'][2]) print('True all: ', fit_config_dict['model_parameter_means']) # get likleihood from fit - T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results) + T.hist = T.TritiumSpectrumBackground(T.bin_centers, *modified_results) fit_ll = T.PoissonLogLikelihood(results_true_mass) # get likleihood of asimov best mass #T.hist = T.TritiumSpectrumBackground(T.bin_centers, *results_best_mass) From e71f3b19e6842597800a82dcfcb1fade6ea31a9c Mon Sep 17 00:00:00 2001 From: cclaessens Date: Fri, 20 Jan 2023 21:23:36 -0800 Subject: [PATCH 193/199] using prior means again for likelihood calculation --- .../processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index 93524e64..bae33b25 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -122,8 +122,13 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, return results, T.minos_errors, total_counts elif 'return_ll' in fit_config_dict and fit_config_dict['return_ll']: - modified_results = deepcopy(results) - modified_results[1] = 0 + modified_results = deepcopy(fit_config_dict['model_parameter_means']) + modified_results[2] = results[2] + + # this works + #modified_results = deepcopy(results) + #modified_results[1] = 0 + all_params = modified_results #all_params = fit_config_dict['model_parameter_means'] From 2a46342e1b2d4b0557e3be009ac44d041e6d4b9f Mon Sep 17 00:00:00 2001 From: cclaessens Date: Tue, 7 Feb 2023 09:18:21 -0800 Subject: [PATCH 194/199] back to setting background 0 in LL ratio calculation --- .../processors/TritiumSpectrum/BinnedTritiumMLFitter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py index bae33b25..85e37249 100755 --- a/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py +++ b/mermithid/processors/TritiumSpectrum/BinnedTritiumMLFitter.py @@ -122,12 +122,12 @@ def DoOneFit(data, fit_config_dict, sampled_parameters={}, error_scaling=0, return results, T.minos_errors, total_counts elif 'return_ll' in fit_config_dict and fit_config_dict['return_ll']: - modified_results = deepcopy(fit_config_dict['model_parameter_means']) - modified_results[2] = results[2] + #modified_results = deepcopy(fit_config_dict['model_parameter_means']) + #modified_results[2] = results[2] # this works - #modified_results = deepcopy(results) - #modified_results[1] = 0 + modified_results = deepcopy(results) + modified_results[1] = 0 all_params = modified_results #all_params = fit_config_dict['model_parameter_means'] From 26cb45e584c010b8f09315398cbae1d85155092b Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 24 Apr 2023 15:57:52 -0700 Subject: [PATCH 195/199] Updated Cicada (v1.4.1) and Phylloxera (v1.3.1) --- Cicada | 2 +- Phylloxera | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cicada b/Cicada index eac4c64a..66779bac 160000 --- a/Cicada +++ b/Cicada @@ -1 +1 @@ -Subproject commit eac4c64adbcd6412912599ee754a953d01b4a860 +Subproject commit 66779bacb7cab84fa75a103a5ef7087bb87e960a diff --git a/Phylloxera b/Phylloxera index 5d2ce40b..908f83c4 160000 --- a/Phylloxera +++ b/Phylloxera @@ -1 +1 @@ -Subproject commit 5d2ce40b3e7fd65d17c5506169903ecb0cb94c63 +Subproject commit 908f83c4cccc7c1a02d4afe87a4742f29dfbec1d From 2d1f81873c2dcdd6ed23c2a417f165949e29a38e Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 24 Apr 2023 15:58:36 -0700 Subject: [PATCH 196/199] Update Scarab and Cicada option setting --- CMakeLists.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 17ca821f..78cf125b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,13 +9,13 @@ include( PackageBuilder ) pbuilder_prepare_project() +set_option( Scarab_BUILD_CODEC_JSON OFF ) +set_option( Scarab_BUILD_CODEC_YAML OFF ) +set_option( Scarab_BUILD_AUTHENTICATION OFF ) +set_option( Scarab_BUILD_CLI OFF ) +set_option( Scarab_BUILD_PARAM OFF ) +set_option( Scarab_ENABLE_EXECUTABLES OFF ) +set_option( Cicada_ENABLE_KATYDID_NAMESPACE OFF ) + pbuilder_add_submodule( Cicada Cicada ) pbuilder_add_submodule( Phylloxera Phylloxera ) - -set( Scarab_BUILD_CODEC_JSON OFF CACHE BOOL "No coded" FORCE) -set( Scarab_BUILD_CODEC_YAML OFF CACHE BOOL "No codec" FORCE) -set( Scarab_BUILD_AUTHENTICATION OFF CACHE BOOL "No auth" FORCE) -set( Scarab_BUILD_PARAM OFF CACHE BOOL "No param" FORCE) - -set( Cicada_ENABLE_KATYDID_NAMESPACE OFF CACHE BOOL "Use Cicada namespace" FORCE) - From c7b87038184eff15a08425a05bd60b1f1d5ee6ea Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 24 Apr 2023 15:58:52 -0700 Subject: [PATCH 197/199] Don't update pip3 (some version shear problem) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e0e8c544..3599cffa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ COPY tests $MERMITHID_BUILD_PREFIX/tests # repeat the cmake command to get the change of install prefix to set correctly (a package_builder known issue) RUN source $MERMITHID_BUILD_PREFIX/setup.sh &&\ - pip3 install --upgrade pip &&\ +# pip3 install --upgrade pip &&\ cd /tmp_source &&\ mkdir -p build &&\ cd build &&\ From c7ab6c3e8a62e5a519aa7a79e9bee542991c462a Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 24 Apr 2023 16:03:41 -0700 Subject: [PATCH 198/199] Updated .gitignore to exclude build directories --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 80b79d73..75a103c6 100644 --- a/.gitignore +++ b/.gitignore @@ -101,3 +101,6 @@ ENV/ .mypy_cache/ *.root *.png + +# CMake build directory +build*/ From d914a879935441bfa76a9d382d327823e3fcac7a Mon Sep 17 00:00:00 2001 From: taliaweiss Date: Wed, 21 Jan 2026 20:45:58 -0500 Subject: [PATCH 199/199] Explained source/calculation of numbers in aseev_func_tail Added link to database with oscillator strengths and explained why data was fitted to produce the values in aseev_func_tail --- mermithid/misc/ComplexLineShapeUtilities.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mermithid/misc/ComplexLineShapeUtilities.py b/mermithid/misc/ComplexLineShapeUtilities.py index 0ff72f2b..4566ac58 100644 --- a/mermithid/misc/ComplexLineShapeUtilities.py +++ b/mermithid/misc/ComplexLineShapeUtilities.py @@ -53,9 +53,14 @@ def read_oscillator_str_file(filename): energyOsc[1] = energyOsc[1][sorted_indices] return energyOsc -# A sub function for the scatter function. Found in +# A sub function for the scatter function. The oscillator strength tails are from +# this database: https://nl.lxcat.net/home/. +# To get the parameters below, the tails are then fitted using the function in: # "Energy loss of 18 keV electrons in gaseous T and quench condensed D films" # by V.N. Aseev et al. 2000 +# The tails in the LXCAT database sometimes only go out to a certain energy. +# Sometimes, we need energies extending beyond that value. That is why we +# need to use this parameterized form of the tail. def aseev_func_tail(energy_loss_array, gas_type): if gas_type=="H2": A2, omeg2, eps2 = 0.195, 14.13, 10.60 @@ -209,4 +214,4 @@ def shake_spectrum(self): x_array = flip_array(x_array) shake_spectrum = self.full_shake_spectrum(x_array, 0, 24) return shake_spectrum - ############################################################################### \ No newline at end of file + ###############################################################################