import sys
import olx
import OlexVFS
import time
from History import *
from my_refine_util import *
from PeriodicTable import PeriodicTable
import olexex
try:
olx.current_reflection_filename
except:
olx.current_reflection_filename = None
olx.current_reflections = None
import olex
import time
import cctbx_controller as cctbx_controller
from cctbx import uctbx
from olexFunctions import OlexFunctions
OV = OlexFunctions()
class OlexRefinementModel(object):
def __init__(self):
olex_refinement_model = OV.GetRefinementModel(True)
self._atoms = {}
self.id_for_name = {}
asu = olex_refinement_model['aunit']
for residue in asu['residues']:
for atom in residue['atoms']:
i = atom['tag']
self._atoms.setdefault(i, atom)
element_type = atom['type']
self.id_for_name.setdefault(str(atom['label']), atom['aunit_id'])
self._cell = olex_refinement_model['aunit']['cell']
self.exptl = olex_refinement_model['exptl']
self._afix = olex_refinement_model['afix']
self.model= olex_refinement_model
def iterator(self):
for i, atom in self._atoms.items():
name = str(atom['label'])
element_type = str(atom['type'])
xyz = atom['crd'][0]
occu = atom['occu']
adp = atom.get('adp',None)
if adp is None:
uiso = atom.get('uiso')[0]
u = (uiso,)
else: u = adp[0]
if name[:1] != "Q":
yield name, xyz, occu, u, element_type
def afix_iterator(self):
for afix in self._afix:
mn = afix['afix']
m, n = divmod(mn, 10)
pivot = afix['pivot']
dependent = afix['dependent']
pivot_neighbours = [i for i in self._atoms[pivot]['neighbours'] if not i in dependent]
bond_length = afix['d']
uiso = afix['u']
yield m, n, pivot, dependent, pivot_neighbours, bond_length
def restraint_iterator(self):
for restraint_type in ('dfix','dang','flat','chiv'):
for restraint in self.model[restraint_type]:
kwds = dict(
i_seqs = [i[0] for i in restraint['atoms']],
sym_ops = [i[1] for i in restraint['atoms']],
value = restraint['value'],
weight = 1/math.pow(restraint['esd1'],2),
)
yield restraint_type, kwds
def getCell(self):
return [self._cell[param][0] for param in ('a','b','c','alpha','beta','gamma')]
class OlexCctbxAdapter(object):
def __init__(self):
if OV.HasGUI():
sys.stdout.refresh = True
self._xray_structure = None
self.olx_atoms = OlexRefinementModel()
self.wavelength = self.olx_atoms.exptl.get('radiation', 0.71073)
try:
self.reflections = None
self.initialise_reflections()
except Exception, err:
sys.stderr.formatExceptionInfo()
def __del__(self):
sys.stdout.refresh = False
def xray_structure(self):
if self._xray_structure is not None:
return self._xray_structure
else:
return cctbx_controller.create_cctbx_xray_structure(
self.cell,
self.space_group,
self.olx_atoms.iterator(),
restraint_iterator=self.olx_atoms.restraint_iterator())
def initialise_reflections(self, force=False, verbose=False):
self.cell = self.olx_atoms.getCell()
self.space_group = str(olx.xf_au_GetCellSymm())
reflections = olx.HKLSrc()
if force or reflections != olx.current_reflection_filename:
olx.current_reflection_filename = reflections
olx.current_reflections = cctbx_controller.reflections(self.cell, self.space_group, reflections)
if olx.current_reflections:
self.reflections = olx.current_reflections
else:
olx.current_reflections = cctbx_controller.reflections(self.cell, self.space_group, reflections)
self.reflections = olx.current_reflections
merge = self.olx_atoms.model.get('merge')
if force or merge is None or merge != self.reflections._merge:
self.reflections.merge(merge=merge)
omit = self.olx_atoms.model['omit']
if force or omit is None or omit != self.reflections._omit:
self.reflections.filter(omit, self.olx_atoms.exptl['radiation'])
if verbose:
self.reflections.show_summary()
class OlexCctbxRefine(OlexCctbxAdapter):
def __init__(self, max_cycles=None, verbose=False):
OlexCctbxAdapter.__init__(self)
self.verbose = verbose
self.log = open('%s/%s.log' %(OV.FilePath(), OV.FileName()),'w')
PT = PeriodicTable()
self.pt = PT.PeriodicTable()
self.olx = olx
self.cycle = 0
self.tidy = False
self.auto = False
self.debug = False
self.film = False
self.max_cycles = max_cycles * 10
self.do_refinement = True
if OV.HasGUI() and OV.GetParam('snum.refinement.graphical_output'):
import Analysis
self.plot = Analysis.smtbx_refine_graph()
else: self.plot = None
def run(self):
t0 = time.time()
print "++++ Refining using the CCTBX with a maximum of %i cycles++++" %self.max_cycles
self.refine_with_cctbx()
asu_mappings = self.xray_structure().asu_mappings(buffer_thickness=3.5)
#
scatterers = self.xray_structure().scatterers()
scattering_types = scatterers.extract_scattering_types()
pair_asu_table = crystal.pair_asu_table(asu_mappings=asu_mappings)
pair_asu_table.add_covalent_pairs(
scattering_types, exclude_scattering_types=flex.std_string(("H","D")))
pair_sym_table = pair_asu_table.extract_pair_sym_table()
print >> self.log, "\nConnectivity table"
pair_sym_table.show(site_labels=scatterers.extract_labels(), f=self.log)
print >> self.log, "\nBond lengths"
self.xray_structure().show_distances(pair_asu_table=pair_asu_table, out=self.log)
print >> self.log, "\nBond angles"
self.xray_structure().show_angles(pair_asu_table=pair_asu_table, out=self.log)
#
self.post_peaks(self.refinement.f_obs_minus_f_calc_map(0.4))
self.log.close()
#cctbxmap_resolution = 0.4
#cctbxmap_type = OV.FindValue('snum_cctbx_map_type', None)
#if cctbxmap_type == "--":
#cctbxmap_type = None
#else:
#cctbxmap_resolution = float(OV.FindValue('snum_cctbxmap_resolution'))
#if cctbxmap_type and cctbxmap_type !='None':
#self.write_grid_file(cctbxmap_type, cctbxmap_resolution)
print "++++ Finished in %.3f s" %(time.time() - t0)
print "Done."
def refine_with_cctbx(self):
wavelength = self.olx_atoms.exptl.get('radiation', 0.71073)
weight = self.olx_atoms.model['weight']
params = dict(a=0.1, b=0, c=0, d=0, e=0, f=1./3)
for param, value in zip(params.keys()[:len(weight)], weight):
params[param] = value
weighting = xray.weighting_schemes.shelx_weighting(**params)
self.reflections.show_summary(log=self.log)
self.refinement = cctbx_controller.refinement(
f_sq_obs=self.reflections.f_sq_obs_merged,
xray_structure=self.xray_structure(),
wavelength=wavelength,
max_cycles=self.max_cycles,
log=self.log,
weighting=weighting,
)
self.refinement.on_cycle_finished = self.feed_olex
self.refinement.start()
self.R1 = self.refinement.R1()[0]
self.wR2, self.GooF = self.refinement.wR2_and_GooF(
self.refinement.minimisation.minimizer)
self.update_refinement_info("Starting...")
def feed_olex(self, structure, minimisation, minimiser):
self.auto = False
self.cycle += 1
#msg = "Refinement Cycle: %i" %(self.cycle)
self.refinement.show_cycle_summary(minimiser)
#print msg
#self.update_refinement_info(msg=msg)
minimisation.show_sorted_shifts(max_items=10, log=self.log)
max_shift_site, max_shift_site_scatterer = minimisation.iter_shifts_sites(max_items=1).next()
print "Max. shift: %.3f %s" %(max_shift_site, max_shift_site_scatterer.label)
if not minimisation.pre_minimisation:
max_shift_u, max_shift_u_scatterer = minimisation.iter_shifts_u(max_items=1).next()
print "Max. dU: %.3f %s" %(max_shift_u, max_shift_u_scatterer.label)
if self.plot is not None:
self.plot.observe(max_shift_site, max_shift_site_scatterer)
if self.film:
n = str(self.cycle)
if int(n) < 10:
n = "0%s" %n
olx.Picta(r"%s0%s.bmp 1" %(self.film, n))
reset_refinement = False
## Feed Model
u_total = 0
u_atoms = []
i = 1
for name, xyz, u, ueq, symbol in self.refinement.iter_scatterers():
if len(u) == 6:
u_trans = (u[0], u[1], u[2], u[5], u[4], u[3])
else:
u_trans = u
id = self.olx_atoms.id_for_name[name]
olx.xf_au_SetAtomCrd(id, *xyz)
olx.xf_au_SetAtomU(id, *u_trans)
u_total += u[0]
if self.tidy:
if u[0] > 0.09:
olx.Name("%s Q" %name)
u_average = u_total/i
if reset_refinement:
raise Exception("Atoms with SillyU Deleted")
if self.auto:
for name, xyz, u, symbol in self.refinement.iter_scatterers():
if symbol == 'H': continue
id = self.olx_atoms.id_for_name[name]
selbst_currently_present = curr_formula.get(symbol, 0)
#print name, u[0]
if u[0] < u_average * 0.8:
# print " ------> PROMOTE?"
promote_to = element_d[symbol].get('+1', symbol)
currently_present = curr_formula.get(promote_to, 0)
max_possible = element_d[promote_to].get('max_number')
if self.debug: olx.Sel(name)
if self.debug: print "Promote %s to a %s. There are %.2f present, and %.2f are allowed" %(name, promote_to, currently_present, max_possible),
if currently_present < max_possible:
olx.xf_au_SetAtomlabel(id, promote_to)
curr_formula[promote_to] = currently_present + 1
curr_formula[symbol] = selbst_currently_present - 1
if self.debug: print " OK"
else:
if self.debug: print " X"
if u[0] > u_average * 1.5:
# print " DEMOTE? <-------"
#reset_refinement = True
demote_to = element_d[symbol].get('-1', symbol)
currently_present = curr_formula.get(demote_to, 0)
max_possible = element_d[demote_to].get('max_number')
if self.debug: olx.Sel(name)
if self.debug: print "Demote %s to a %s. There are %.2f present, and %.2f are allowed" %(name, demote_to, currently_present, max_possible),
if curr_formula.get(demote_to, 0) < element_d[demote_to].get('max_number'):
olx.xf_au_SetAtomlabel(id, demote_to)
curr_formula[demote_to] = currently_present + 1
curr_formula[symbol] = selbst_currently_present - 1
if self.debug: print "OK"
else:
if self.debug: print " X"
olx.Sel('-u')
olx.xf_EndUpdate()
if reset_refinement:
raise Exception("Atoms promoted")
def post_peaks(self, fft_map):
from cctbx import maptbx
from libtbx.itertbx import izip
fft_map.apply_volume_scaling()
peaks = fft_map.peak_search(
parameters=maptbx.peak_search_parameters(
peak_search_level=3,
interpolate=False,
#peak_cutoff=0.1,
min_distance_sym_equiv=1.0,
max_clusters=30,
),
verify_symmetry=False
).all()
#peaks = self.refinement.peak_search()
#peaks.show_sorted()
i = 0
for xyz, height in izip(peaks.sites(), peaks.heights()):
if i < 3:
if self.verbose: print "Position of peak %s = %s, Height = %s" %(i, xyz, height)
i += 1
id = olx.xf_au_NewAtom("%.2f" %(height), *xyz)
if id != '-1':
olx.xf_au_SetAtomU(id, "0.06")
if i == 100:
break
olx.xf_EndUpdate()
olx.Compaq('-a')
olx.Refresh()
def update_refinement_info(self, msg="Unknown"):
import htmlMaker
R1, n_reflections = self.refinement.R1()
R1_all_data, n_reflections_unique = self.refinement.R1(all_data=True)
wR2 = self.wR2
GooF = self.GooF
#htmlMaker.make_refinement_data_html(R1=R1,
#n_reflections=n_reflections,
#R1_all_data=R1_all_data,
#n_reflections_unique=n_reflections_unique,
#wR2=wR2,
#GooF=GooF)
txt = "Last refinement with smtbx-refine: No refinement info available yet.
Status: %s" %msg
OlexVFS.write_to_olex('refinedata.htm', txt)
def write_grid_file(self, type, resolution):
import olex_xgrid
if type == "DIFF":
m = self.refinement.get_difference_map(resolution)
elif type == "FOBS":
m = self.refinement.get_f_obs_map(resolution)
else:
return
s = m.last()
olex_xgrid.Init(s[0], s[1], s[2])
for i in range (s[0]-1):
for j in range (s[1]-1):
for k in range (s[2]-1):
olex_xgrid.SetValue( i,j,k,m[i,j,k])
olex_xgrid.InitSurface(True)
class OlexCctbxSolve(OlexCctbxAdapter):
def __init__(self):
OlexCctbxAdapter.__init__(self)
self.peak_normaliser = 1200 #fudge factor to get cctbx peaks on the same scale as shelx peaks
def runChargeFlippingSolution(self, hkl_path, verbose='highly', solving_interval=60):
import time
t1 = time.time()
from smtbx.ab_initio import charge_flipping
from cctbx import maptbx
from libtbx.itertbx import izip
t2 = time.time()
print 'imports took %0.3f ms' %((t2-t1)*1000.0)
# Get the reflections from the specified path
f_obs = self.reflections.f_obs
data = self.reflections.f_sq_obs
# merge them (essential!!)
merging = f_obs.merge_equivalents()
f_obs = merging.array()
f_obs.show_summary()
# charge flipping iterations
#flipping = charge_flipping.basic_iterator(f_obs, delta=None)
flipping = charge_flipping.weak_reflection_improved_iterator(f_obs, delta=None)
solving = charge_flipping.solving_iterator(
flipping,
yield_during_delta_guessing=True,
yield_solving_interval=solving_interval)
charge_flipping_loop(solving, verbose=verbose)
# play with the solutions
expected_peaks = data.unit_cell().volume()/18.6/len(data.space_group())
expected_peaks *= 1.3
if solving.f_calc_solutions:
# actually only the supposedly best one
f_calc, shift, cc_peak_height = solving.f_calc_solutions[0]
fft_map = f_calc.fft_map(
symmetry_flags=maptbx.use_space_group_symmetry)
# search and print Fourier peaks
peaks = fft_map.peak_search(
parameters=maptbx.peak_search_parameters(
min_distance_sym_equiv=1.0,
max_clusters=expected_peaks,),
verify_symmetry=False
).all()
for q,h in izip(peaks.sites(), peaks.heights()):
yield q,h
#print "(%.3f, %.3f, %.3f) -> %.3f" % (q+(h,))
else:
print "*** No solution found ***"
def post_single_peak(self, xyz, height, cutoff=1.0, auto_assign=False):
if height/self.peak_normaliser < cutoff:
return
sp = (height/self.peak_normaliser)
if not auto_assign:
id = olx.xf_au_NewAtom("%.2f" %(sp), *xyz)
#if olx.xf_au_SetAtomCrd(id, *xyz)=="true":
if id != '-1':
olx.xf_au_SetAtomU(id, "0.06")
else:
max_Z = 0
for element in auto_assign:
Z = self.pt[element].get('Z', 0)
if self.pt[element].get('Z', 0) > max_Z: max_Z = Z
please_assign = False
for element in auto_assign:
if element == "H":
continue
Z = self.pt[element].get('Z', 0)
if (sp-sp*0.1) < Z < (sp + sp*0.1):
please_assign = True
break
elif sp > Z and Z == max_Z:
please_assign = True
break
if please_assign:
id = olx.xf_au_NewAtom(element, *xyz)
#olx.xf_au_SetAtomCrd(id, *xyz)
#olx.xf_au_SetAtomOccu(id, 1)
auto_assign[element]['count'] -= 1
if auto_assign[element]['count'] == 0:
del auto_assign[element]
return
else:
if sp > 2:
id = olx.xf_au_NewAtom("C", *xyz)
#olx.xf_au_SetAtomCrd(id, *xyz)
return
id = olx.xf_au_NewAtom("%.2f" %(sp), *xyz)
#if olx.xf_au_SetAtomCrd(id, *xyz)=="true":
if id != '-1':
olx.xf_au_SetAtomU(id, "0.06")
class OlexCctbxGraphs(OlexCctbxAdapter):
def __init__(self, graph, *args, **kwds):
OlexCctbxAdapter.__init__(self)
if self.reflections is None:
raise RuntimeError, "There was an error reading the reflection file."
self.graph = graph
bitmap = 'working'
OV.CreateBitmap(bitmap)
try:
if graph == "wilson":
self.xy_plot = cctbx_controller.wilson_statistics(self.xray_structure(), self.reflections, **kwds)
elif graph == "completeness":
self.xy_plot = cctbx_controller.completeness_statistics(self.reflections, self.wavelength, **kwds)
elif graph == "cumulative":
self.xy_plot = cctbx_controller.cumulative_intensity_distribution(self.reflections, **kwds)
elif graph == "f_obs_f_calc":
twinning = self.olx_atoms.model.get('twin')
self.xy_plot = cctbx_controller.f_obs_vs_f_calc(self.xray_structure(), self.reflections, twinning=twinning, **kwds)
elif graph == "f_obs_over_f_calc":
self.xy_plot = cctbx_controller.f_obs_over_f_calc(self.xray_structure(), self.reflections, self.wavelength, **kwds)
elif graph == "sys_absent":
self.xy_plot = cctbx_controller.sys_absent_intensity_distribution(self.reflections)
elif graph == "normal_probability":
weight = self.olx_atoms.model['weight']
params = dict(a=0.1, b=0, c=0, d=0, e=0, f=1./3)
for param, value in zip(params.keys()[:len(weight)], weight):
params[param] = value
weighting = xray.weighting_schemes.shelx_weighting(**params)
self.xy_plot = cctbx_controller.normal_probability_plot(
self.xray_structure(), self.reflections, weighting,
#distribution=cctbx_controller.distributions.students_t_distribution(5),
).xy_plot_info()
except Exception, err:
raise Exception, err
finally:
OV.DeleteBitmap(bitmap)
class OlexCctbxTwinLaws(OlexCctbxAdapter):
def __init__(self):
OlexCctbxAdapter.__init__(self)
self.run()
def run(self):
from PilTools import MatrixMaker
a = MatrixMaker()
twin_laws = cctbx_controller.twin_laws(self.reflections)
r_list = []
l = 0
self.twin_law_gui_txt = ""
if not twin_laws:
print "There are no possible twin laws"
html = "