#!/bin/env python

# written 2019-01-28 by mza
# last updated 2019-11-04 by mza

import threading # Thread()
from B2L_common import *
from B2L_defs import *
from get_status import get_status
import ethudp
from DebugInfoWarningError24 import debug, info, warning, error, debug2, debug3, set_verbosity
import time # sleep()

BS = 0
set_verbosity(3)
frame_mask_shift = 16
frame_mask_all = 0b111111111 << frame_mask_shift
bunch_marker_first_valid = 4
initial_bunch_marker_of_interest = 1701
bunch_marker_last_valid = 5120 - 1
#number_of_64_sample_windows_per_beam_orbit = 427

def set_trigger_mask(s, c0, c1, c2, c3):
	write_register(SCROD_AxiCommon_trigMask, s, BS)
	write_register(CARRIER_AxiCommon_trigMask, c0, BS, 0)
	write_register(CARRIER_AxiCommon_trigMask, c1, BS, 1)
	write_register(CARRIER_AxiCommon_trigMask, c2, BS, 2)
	write_register(CARRIER_AxiCommon_trigMask, c3, BS, 3)

def mask_triggers(s=0xf, c0=0xf, c1=0xf, c2=0xf, c3=0xf):
	set_trigger_mask(s, c0, c1, c2, c3)

def unmask_triggers(s=0, c0=0, c1=0, c2=0, c3=0):
	set_trigger_mask(s, c0, c1, c2, c3)

def set_trig_prescale_N_log2(N_log2):
	write_register(SCROD_AxiCommon_trig_prescale_N_log2, N_log2, BS)

def get_xrm_trigger_enable_status():
	return read_register(SCROD_AxiCommon_xrm_trigger_enabled, BS)

def enable_xrm_triggers():
	write_register(SCROD_AxiCommon_xrm_trigger_enabled, 1, BS)

def disable_xrm_triggers():
	write_register(SCROD_AxiCommon_xrm_trigger_enabled, 0, BS)

def allow_xrm_triggers_for_a_while(duration=1.0):
	if be_verbose_when_collecting_data:
		info("allowing xrm triggers for " + str(int(1000.0*duration)/1000.0) + " seconds...")
	a = get_SCROD_eventCnt()
	enable_xrm_triggers()
	time.sleep(duration)
	disable_xrm_triggers()
	b = get_SCROD_eventCnt()
	diff = b - a
	return diff

def trigger_and_wait_for_xrm_triggers(duration=1.0):
#	if be_verbose_when_collecting_data:
#		info("allowing xrm triggers for " + str(int(1000.0*duration)/1000.0) + " seconds...")
	a = get_SCROD_eventCnt()
	if should_software_trigger:
		software_trigger(BS)
	else:
		write_register_without_readback_verification(SCROD_AxiCommon_clear_count_of_triggers_for_bunch_markers, 0xf, BS)
	wait_at_least_until_next_frame9()
	time.sleep(duration)
	b = get_SCROD_eventCnt()
	diff = b - a
	return diff

expected_trigger_rate = { 0:11150.0, 1:5560.0, 2:2780.0, 3:1390.0, 4:690.0, 5:350.0, 6:170.0, 7:90.0, 8:40.0, 9:20.0, 10:10.0, 11:5.0, 12:2.0, 13:1.0, 14:1.0, 15:1.0, 16:1.0, 17:1.0, 18:1.0 }
def generate_histogram_of_trigger_rate_versus_prescale_N_log2(start, end):
	global expected_trigger_rate
	old_desired_trigger_quantities = set_desired_trigger_quantities(0, 0, 0, 0)
	mask_triggers()
	#max_rate = 151000.0
	for i in range(start, end+1):
		set_trig_prescale_N_log2(i)
		#duration = 40.0 * (2.0**i) / max_rate
		duration = 0.1
		#info(duration)
		if i > 10:
			duration = 1.0
		diff = allow_xrm_triggers_for_a_while(duration)
		#info(str(diff))
		rate = diff / duration
		expected_trigger_rate[i] = rate
		info("[" + str(i) + "]: " + str(rate))
	set_desired_trigger_quantities(old_desired_trigger_quantities[0], old_desired_trigger_quantities[1], old_desired_trigger_quantities[2], old_desired_trigger_quantities[3])

def get_SCROD_eventCnt(should_show_it=0):
	SCROD_eventCnt = read_register(SCROD_AxiCommon_eventCnt, BS)
	if should_show_it:
		info("SCROD_eventCnt: " + str(SCROD_eventCnt))
	return SCROD_eventCnt

def get_carrier_eventCnts(should_show_them=0):
	if 5==number_of_carriers[BS] or 0==number_of_carriers[BS]:
		return
	eventCnt = []
	if 1 <= number_of_carriers[BS]:
		carrier0_eventCnt = read_register(CARRIER_AxiCommon_eventCnt, BS, 0)
		eventCnt.append(carrier0_eventCnt)
		if should_show_them:
			info("carrier0_eventCnt: " + str(carrier0_eventCnt))
	if 2 <= number_of_carriers[BS]:
		carrier1_eventCnt = read_register(CARRIER_AxiCommon_eventCnt, BS, 1)
		eventCnt.append(carrier1_eventCnt)
		if should_show_them:
			info("carrier1_eventCnt: " + str(carrier1_eventCnt))
	if 3 <= number_of_carriers[BS]:
		carrier2_eventCnt = read_register(CARRIER_AxiCommon_eventCnt, BS, 2)
		eventCnt.append(carrier2_eventCnt)
		if should_show_them:
			info("carrier2_eventCnt: " + str(carrier2_eventCnt))
	if 4 <= number_of_carriers[BS]:
		carrier3_eventCnt = read_register(CARRIER_AxiCommon_eventCnt, BS, 3)
		eventCnt.append(carrier3_eventCnt)
		if should_show_them:
			info("carrier3_eventCnt: " + str(carrier3_eventCnt))
	return eventCnt

def set_readout_window_to_match_bunch(bunch):
	write_register_for_each_asic(CARRIER_IRSX_readoutLookback, int(bunch/RF_buckets_per_read_window), BS)

def set_bunch_marker(bunch_marker_number, frame_mask, value):
	set_readout_window_to_match_bunch(value)
	value &= ~frame_mask_all
	value |= frame_mask << frame_mask_shift
	#info(hex(value))
	if 0==bunch_marker_number:
		write_register(SCROD_AxiCommon_bunch_marker_a, value, BS)
	if 1==bunch_marker_number:
		write_register(SCROD_AxiCommon_bunch_marker_b, value, BS)
	if 2==bunch_marker_number:
		write_register(SCROD_AxiCommon_bunch_marker_c, value, BS)
	if 3==bunch_marker_number:
		write_register(SCROD_AxiCommon_bunch_marker_d, value, BS)

#def set_bunch_markers(a, b, c, d):
#	a |= frame_mask_all
#	b |= frame_mask_all
#	c |= frame_mask_all
#	d |= frame_mask_all
#	#a &= ~frame_mask_all
#	#b &= ~frame_mask_all
#	#c &= ~frame_mask_all
#	#d &= ~frame_mask_all
#	#a |= (1<<0)<<frame_mask_shift
#	#b |= (1<<5)<<frame_mask_shift
#	#c |= (1<<6)<<frame_mask_shift
#	#d |= (1<<7)<<frame_mask_shift
#	write_register(SCROD_AxiCommon_bunch_marker_a, a, BS)
#	write_register(SCROD_AxiCommon_bunch_marker_b, b, BS)
#	write_register(SCROD_AxiCommon_bunch_marker_c, c, BS)
#	write_register(SCROD_AxiCommon_bunch_marker_d, d, BS)
#
#def set_quadbunch_markers(a, b, c, d):
#	set_bunch_markers(4*a, 4*b, 4*c, 4*d)
#
#def set_asic_write_window_bunch_markers(a, b, c, d):
#	#calculate_parameters(4)
#	#calculate_parameters(6)
#	#calculate_parameters(8)
#	#set_bunch_markers(16*a, 16*b, 16*c, 16*d)
#	set_bunch_markers(24*a, 24*b, 24*c, 24*d)
#	#set_bunch_markers(32*a, 32*b, 32*c, 32*d)

def set_desired_trigger_quantities(a, b, c, d):
	old_a = read_register(SCROD_AxiCommon_desired_trigger_quantity_for_bunch_marker_a, BS)
	old_b = read_register(SCROD_AxiCommon_desired_trigger_quantity_for_bunch_marker_b, BS)
	old_c = read_register(SCROD_AxiCommon_desired_trigger_quantity_for_bunch_marker_c, BS)
	old_d = read_register(SCROD_AxiCommon_desired_trigger_quantity_for_bunch_marker_d, BS)
	write_register(SCROD_AxiCommon_desired_trigger_quantity_for_bunch_marker_a, a, BS)
	write_register(SCROD_AxiCommon_desired_trigger_quantity_for_bunch_marker_b, b, BS)
	write_register(SCROD_AxiCommon_desired_trigger_quantity_for_bunch_marker_c, c, BS)
	write_register(SCROD_AxiCommon_desired_trigger_quantity_for_bunch_marker_d, d, BS)
	return old_a, old_b, old_c, old_d

def calculate_parameters(SST_clock_divide_ratio=6, should_print_info=0):
	# 5120 / 4 = 1280 FTSW clocks per frame
	# 5120 buckets / 508.8875 MHz = 10.0611628307 us per frame
	#
	# 127.221875 MHz / 6 = 21.203645833 MHz
	# 128 samples per write window * 21.203645833 MHz SST = 2714.066666 MHz sampling rate
	# 1/21.203645833 MHz = 47.1617007689 ns per write window
	# 10.0611628307 us / 47.1617007689 ns = 213.333333 write windows per revolution
	# 5120 / 213.333333 = 24.0 bunches per window
	# 6.0 FTSW bunches per window
	#
	# 127.221875 MHz / 8 = 15.902734375 MHz
	# 128 samples per write window * 15.902734375 MHz SST = 2035.55 MHz sampling rate
	# 1/15.902734375 MHz = 62.882268 ns per write window
	# 10.0611628307 us / 62.882268 ns = 160 write windows per revolution
	# 5120 / 160 = 32.0 bunches per window
	# 8.0 FTSW bunches per window
	global RF_clock_Hz
	RF_clock_Hz = 508.8875e6
	global period_of_RF_clock_ns
	period_of_RF_clock_ns = 1.0e9 / RF_clock_Hz
	global RF_buckets
	RF_buckets = 5120
	global asic_samples_per_write_window
	asic_samples_per_write_window = 128
	global FTSW_clock_divide_ratio
	FTSW_clock_divide_ratio = 4
	global revolution_duration_s
	revolution_duration_s = RF_buckets / RF_clock_Hz
	global SST_clock_Hz
	SST_clock_Hz = RF_clock_Hz / FTSW_clock_divide_ratio / SST_clock_divide_ratio
	global sampling_rate_Hz
	sampling_rate_Hz = SST_clock_Hz * asic_samples_per_write_window
	global write_window_duration_s
	write_window_duration_s = 1.0 / SST_clock_Hz
	global read_window_duration_s
	read_window_duration_s = write_window_duration_s / 2.0
	global write_windows_per_revolution
	write_windows_per_revolution = revolution_duration_s / write_window_duration_s
	global read_windows_per_revolution
	read_windows_per_revolution = revolution_duration_s / read_window_duration_s
	global RF_buckets_per_write_window
	RF_buckets_per_write_window = RF_buckets / write_windows_per_revolution
	global RF_buckets_per_read_window
	RF_buckets_per_read_window = RF_buckets / read_windows_per_revolution
	global FTSW_clocks_per_write_window
	FTSW_clocks_per_write_window = RF_buckets_per_write_window / FTSW_clock_divide_ratio
	global FTSW_clocks_per_read_window
	FTSW_clocks_per_read_window = RF_buckets_per_read_window / FTSW_clock_divide_ratio
	if should_print_info:
		info("revolution_duration_us: " + str(revolution_duration_s*1.0e6))
		info("SST_clock_MHz: " + str(SST_clock_Hz/1.0e6))
		info("sampling_rate_GHz: " + str(sampling_rate_Hz/1.0e9))
		info("write_window_duration_ns : " + str(write_window_duration_s*1.0e9))
		info("read_window_duration_ns : " + str(read_window_duration_s*1.0e9))
		info("write_windows_per_revolution: " + str(write_windows_per_revolution))
		info("read_windows_per_revolution: " + str(read_windows_per_revolution))
		info("RF_buckets_per_write_window: " + str(RF_buckets_per_write_window))
		info("RF_buckets_per_read_window: " + str(RF_buckets_per_read_window))
		info("FTSW_clocks_per_write_window: " + str(FTSW_clocks_per_write_window))
		info("FTSW_clocks_per_read_window: " + str(FTSW_clocks_per_read_window))

def decode_frame_mask(frame_mask):
	frame_mask &= frame_mask_all
	frame_mask >>= frame_mask_shift
	list = []
	if frame_mask & 0x100:
		list.append(0)
	if frame_mask & 0x080:
		list.append(1)
	if frame_mask & 0x040:
		list.append(2)
	if frame_mask & 0x020:
		list.append(3)
	if frame_mask & 0x010:
		list.append(4)
	if frame_mask & 0x008:
		list.append(5)
	if frame_mask & 0x004:
		list.append(6)
	if frame_mask & 0x002:
		list.append(7)
	if frame_mask & 0x001:
		list.append(8)
	return list

def show_frame_masks():
	a = read_register(SCROD_AxiCommon_bunch_marker_a, BS)
	af = decode_frame_mask(a)
	a &= ~frame_mask_all
	info("a: " + str(af) + " " + str(a))
	b = read_register(SCROD_AxiCommon_bunch_marker_b, BS)
	bf = decode_frame_mask(b)
	b &= ~frame_mask_all
	info("b: " + str(bf) + " " + str(b))
	c = read_register(SCROD_AxiCommon_bunch_marker_c, BS)
	cf = decode_frame_mask(c)
	c &= ~frame_mask_all
	info("c: " + str(cf) + " " + str(c))
	d = read_register(SCROD_AxiCommon_bunch_marker_d, BS)
	df = decode_frame_mask(d)
	d &= ~frame_mask_all
	info("d: " + str(df) + " " + str(d))
	info("total frame masks active: " + str(count_frame_masks()))

def count_frame_masks():
	a = read_register(SCROD_AxiCommon_bunch_marker_a, BS)
	ac = len(decode_frame_mask(a))
	b = read_register(SCROD_AxiCommon_bunch_marker_b, BS)
	bc = len(decode_frame_mask(b))
	c = read_register(SCROD_AxiCommon_bunch_marker_c, BS)
	cc = len(decode_frame_mask(c))
	d = read_register(SCROD_AxiCommon_bunch_marker_d, BS)
	dc = len(decode_frame_mask(d))
	total = ac + bc + cc + dc
	return total

def show_xrm_trigger_enable_status():
	en = read_register(SCROD_AxiCommon_xrm_trigger_enabled, BS)
	info("xrm trigger enabled: " + str(en))

def show_relevant_registers():
	#get_SCROD_eventCnt(1)
	#get_carrier_eventCnts(1)
	show_frame_masks()
	nlog2 = read_register(SCROD_AxiCommon_trig_prescale_N_log2, BS)
	info("N_log2 prescale: " + str(nlog2))
	show_xrm_trigger_enable_status()
	show_asic_timeouts(BS)
	show_SEM_status(BS)

def show_some_info():
	info("")
	show_current_time()
	show_asic_timeouts(BS)
	get_SCROD_eventCnt(1)
	get_carrier_eventCnts(1)
	show_relevant_registers()
	#show_scalers(BS)
	get_status(BS)

def get_mtime(filename):
	return os.stat(filename).st_mtime

def set_mtime(destination, source):
	reftime = get_mtime(source)
	os.utime(destination, (reftime, reftime))
	newtime = get_mtime(destination)
	#info(hex(reftime))
	#info(hex(newtime))

def parse_all_datafiles_in_this_dirtree(dirname, forced=0):
	for root, dirs, files in os.walk(dirname):
		for dirname in sorted(dirs):
			parse_all_datafiles_in_this_dirtree(os.path.join(root, dirname))
		for filename in sorted(files):
			match = re.search("(.*)\.datafile$", filename)
			if match:
				global datafilename
				datafilename = os.path.join(root, filename)
				#info(datafilename)
				global waveformfilename
				waveformfilename = match.group(1) + ".waveforms"
				waveformfilename = os.path.join(root, waveformfilename)
				if forced or not os.path.exists(waveformfilename):
					info("generating " + waveformfilename + "...")
					parse_datafile_and_dump_waveforms_to(waveformfilename)
					close_datafile_for_reading()
					set_mtime(waveformfilename, datafilename)
				#plot_waveforms_from_waveformfile(desired_event_numbers)
				#close_waveformfile_for_reading()

def parse_all_waveformfiles_in_this_dirtree(dirname, forced=0):
	desired_event_numbers = [ -1 ]
	for root, dirs, files in os.walk(dirname):
		#for dirname in dirs:
		for dirname in sorted(dirs):
			parse_all_waveformfiles_in_this_dirtree(os.path.join(root, dirname))
		#for filename in files:
		for filename in sorted(files):
			match = re.search("(.*)\.waveforms$", filename)
			if match:
				global waveformfilename
				waveformfilename = os.path.join(root, filename)
				info("generating plots from " + waveformfilename + "...")
				plot_waveforms_from_waveformfile(desired_event_numbers)
				close_waveformfile_for_reading()

import struct # unpack_from

# https://stackoverflow.com/a/1035456/5728815
def bytes_from_filename(filename, chunksize=8192):
	with open(filename, "rb") as f:
		while True:
			chunk = f.read(chunksize)
			if chunk:
				for b in chunk:
					yield b
			else:
				break

def words_from_filename(filename, chunksize=8192):
	with open(filename, "rb") as f:
		while True:
			try:
				chunk = f.read(chunksize)
			except:
				break
			offset = 0
			if chunk:
				#for w in struct.iter_unpack("I", chunk):
				for i in range(chunksize>>2):
					try:
						w = struct.unpack_from("I", chunk, offset)
						offset += 4
						yield w[0]
					except:
						break
			else:
				break

def open_datafile_for_reading():
	global datafile
	global datafile_event_number
	if datafile is None:
		datafile = open(datafilename, "rb")
		datafile_event_number = 0
#	else:
#		info(str(datafile.tell()))

def open_waveformfile_for_reading():
	global waveformfile
	global waveformfile_event_number
	global waveformfile_waveform_number
	if waveformfile is None:
		waveformfile = open(waveformfilename, "rb")
		waveformfile_event_number = 0
		waveformfile_waveform_number = 0
#	else:
#		info(str(waveformfile.tell()))
	#waveformfile.seek(0)

def close_datafile_for_reading():
	global datafile
	if datafile is not None:
		datafile.close()
		datafile = None

def close_waveformfile_for_reading():
	global waveformfile
	if waveformfile is not None:
		waveformfile.close()
		waveformfile = None

def words_from_file(f, chunksize=16384):
	total_yield = 0
#	seek = 0
	while True:
		try:
			chunk = f.read(chunksize)
		except:
			error("can't read from file")
			break
		offset = 0
		if chunk:
			#for w in struct.iter_unpack("I", chunk):
			for i in range(chunksize>>2):
				try:
					(w,) = struct.unpack_from("I", chunk, offset)
					offset += 4
					total_yield += 1
					yield w
				except:
					#info("breaking at line 398")
					break
		else:
#			time.sleep(1)
#			info("waiting...")
			#info("breaking at line 403")
			break
#		seek += offset
	#info("reached end of file at " + str(f.tell()))
#	total_yield -= 4
	#info("yielded " + str(total_yield) + " words")
	#info("yielded " + str(4*total_yield) + " bytes")
	#yield 0x12345678

def parse_datafile_and_dump_waveforms_to(outfilename):
	#string = 'hexdump datafile -e \'5/4 \"%08X \"\' -e \'\"\\n\"\''
	#info(string)
	#response = commands.getstatusoutput(string)
	#info(response[1])
	#info("")
	if be_verbose_when_collecting_data:
		string = "# t v scr x bonu p cor s cti rev9 mask qu ## "
		info(string)
	# i is word number from Belle 2 TOP data format (production data format 4.1)
	# j is the number of core words remaining
	# k is the number of bonus words remaining
	# n is count of the events in the file so far
	j = 0
	k = 0
	i = 999999
	global datafile_event_number
	open_datafile_for_reading()
	outfile = open(outfilename, "ab")
	info_string = ""
	debug_string = ""
#	old_verbosity = set_verbosity(4)
	just_finished_hits = 0
	waveform = []
	for w in words_from_file(datafile):
		i += 1
		if 0==j and 0==k and 4==(w>>24) and 1==((w>>16)&0xff):
			i = 0
			#outfile.flush()
			if 0!=datafile_event_number:
				if be_verbose_when_collecting_data:
					info(info_string, 0)
					debug(debug_string, 0)
			datafile_event_number += 1
			info_string = str(datafile_event_number) + " "
			debug_string = ""
		if 0==i:
			packet_type = w>>24
			packet_version = (w>>16)&0xff
			should_be_a = (w>>12)&0xf
			scrod_id = w&0xfff
			info_string += hex(packet_type, 1) + " " + hex(packet_version, 1) + " " + hex(scrod_id, 3) + " "
		elif 1==i:
			extra = w>>29
			bonus = (w>>16) & 0x1fff
			k = bonus
			phase = (w>>12) & 0xf
			core = w & 0xfff
			j = core - 5 # core words includes the overall event header
			info_string += hex(extra, 1) + " " + hex(bonus, 4) + " " + hex(phase, 1) + " " + hex(core, 3) + " "
		elif 2==i:
			skipped = w>>31
			should_be_zeroes = (w>>27) & 0xf
			ctime = (w>>16) & 0x7ff
			count_since_revo9 = w & 0xffff
			info_string += hex(skipped, 1) + " " + hex(ctime, 3) + " " + hex(count_since_revo9, 4) + " "
		elif 3==i:
			masks = w>>16
			eventQueueDepth = (w>>8) & 0xff
			eventNumberByte = w & 0xff
			info_string += hex(masks, 4) + " " + hex(eventQueueDepth, 2) + " " + hex(eventNumberByte, 2) + " "
		else: # done with header, go on to hits (interleaved with waveforms if present)
			should_be_b = (w>>12) & 0xf
			#if 0xb==should_be_b:
			#	debug_string += "\n [" + str(j) + "," + str(k) + "] " + hex(w, 8) + " "
			if 0<j:
				if 0==(j%5):
					carrier = (w>>30) & 0x3
					asic    = (w>>28) & 0x3
					channel = (w>>25) & 0x7
					window  = (w>>16) & 0x1ff
					waveform_present = (w>>7) & 1
					if waveform_present:
						waveform_present_string = "w"
					else:
						waveform_present_string = "-"
					if 0xb==should_be_b:
						debug_string += "\n [" + str(j) + "," + str(k) + "] "
						#debug_string += hex(w, 8) + " "
						debug_string += hex(carrier, 1) + " " + hex(asic, 1) + " " + hex(channel, 1) + " " + hex(window, 3) + " " + waveform_present_string + " "
					#else:
					#	debug_string += "\n error parsing hit "
					j -= 1
				elif 4==(j%5):
					j -= 1
				elif 3==(j%5):
					j -= 1
				elif 2==(j%5):
					j -= 1
				elif 1==(j%5):
					j -= 1
				just_finished_hits = 1
			else:
				if just_finished_hits:
					just_finished_hits = 0
					should_be_5 = (w>>9) & 0x7
					if 0x5!=should_be_5:
						debug_string += "error "
				else: # now checkout the waveforms if present
					if 0<k:
						waveform.append(w)
						#outfile.write(struct.pack('I', w))
						#info("wrote to waveform file")
						#outfile.write(struct.pack('IIhhhhhhhhhhhhhhhh', w))
						#if 0==(k%9):
						k -= 1
						if 0==k:
							if len(waveform):
								write_waveform_to_file(outfile, waveform)
								waveform = []
#					else:
#						if len(waveform):
#							outfile.write(struct.pack('I'*len(waveform), *waveform))
#							info("wrote " + str(4*len(waveform)) + " bytes to waveform file")
#							waveform = []
	if len(waveform):
		write_waveform_to_file(outfile, waveform)
	if be_verbose_when_collecting_data:
		info(info_string)
		debug(debug_string)
#	set_verbosity(old_verbosity)
	outfile.close()

def write_waveform_to_file(outfile, waveform):
	if outfile and len(waveform):
		outfile.write(struct.pack('I'*len(waveform), *waveform))
		#info("wrote " + str(4*len(waveform)) + " bytes to waveform file")

number_of_waveform_words_in_a_waveform_packet = 16
def waveforms_from_waveformfile():
	found_valid_first_word = 0
	found_valid_second_word = 0
	mode = "looking_for_start_of_header"
	#mode = "parsing_header"
	#mode = "yielding_data"
	k = 0 # k is the number of waveform words seen so far in each waveform packet
	#old_verbosity = set_verbosity(4)
	if be_verbose_when_collecting_data:
		debug_string = "   c a c samp w e# log phy st"
		debug(debug_string)
	debug_string = ""
	#debug2_string = ""
	array = [ 0 for x in range(2*number_of_waveform_words_in_a_waveform_packet) ]
	global waveformfile_waveform_number
	open_waveformfile_for_reading()
	#words = 0
	#yielding = 0
	#first = 0
	#second = 0
	#did_this = 0
	for w in words_from_file(waveformfile):
		#words += 1
		if "yielding_data"==mode:
			#yielding += 1
			# https://stackoverflow.com/a/6727975/5728815
			one = w>>16
			if (1<<15) & one:
				one -= 1<<16
			zero = w & 0xffff
			if (1<<15) & zero:
				zero -= 1<<16
			#debug2_string += str(zero) + "," + str(one) + " "
			#debug2_string += hex(zero) + "," + hex(one) + " "
			#yield zero
			#yield one
			array[2*k] = zero
			array[2*k+1] = one
			k += 1
			#if 15==k:
			#	info("k=15")
			#if 16==k:
			#	info("k=16")
		elif "parsing_header"==mode and found_valid_first_word:
			#second += 1
			should_be_1_zero_another_one_though = w>>31
			starting_sample = (w>>25) & 0x3f
			logical_window = (w>>16) & 0x1ff
			carrier_event_number_lsbs = (w>>9) & 0x7f
			physical_window = w & 0x1ff
			if 0==should_be_1_zero_another_one_though and logical_window==physical_window and (0==starting_sample or 0x20==starting_sample):
				found_valid_second_word = 1
				debug_string += hex(carrier_event_number_lsbs, 2) + " "
				debug_string += hex(logical_window, 3) + " "
				debug_string += hex(physical_window, 3) + " "
				debug_string += hex(starting_sample, 2) + " "
				if be_verbose_when_collecting_data:
					debug(debug_string)
				debug_string = ""
				array = [ 0 for x in range(2*number_of_waveform_words_in_a_waveform_packet) ]
				k = 0
				mode = "yielding_data"
			else:
				debug_string += "\nerror finding second word " + hex(w, 8)
				#found_valid_first_word = 0
				#found_valid_second_word = 0
				#k = 0
				#array = [ 0 for x in range(2*number_of_waveform_words_in_a_waveform_packet) ]
				mode = "looking_for_start_of_header"
		elif "looking_for_start_of_header"==mode:
			#first += 1
			k = 0
			found_valid_first_word = 0
			found_valid_second_word = 0
			n_samples = w>>16
			should_be_5_zeroes = (w>>11) & 0x1f
			n_windows = (w>>8) & 0x7
			should_be_1_zero = (w>>7) & 1
			carrier = (w>>5) & 0x3
			asic = (w>>3) & 0x3
			channel = w & 0x7
			if 0==should_be_5_zeroes and 0==should_be_1_zero and 0x20==n_samples:
				found_valid_first_word = 1
				waveformfile_waveform_number += 1
				#info("working on the " + str(waveformfile_waveform_number) + "th waveform in the file")
				debug_string = str(waveformfile_waveform_number) + ": "
				debug_string += hex(carrier, 1) + " "
				debug_string += hex(asic, 1) + " "
				debug_string += hex(channel, 1) + " "
				debug_string += hex(n_samples, 2) + " "
				debug_string += hex(n_windows, 1) + " "
				#k = 0
				#array = [ 0 for x in range(2*number_of_waveform_words_in_a_waveform_packet) ]
				mode = "parsing_header"
			else:
				debug_string += "\nerror finding first word " + hex(w, 8)
				#found_valid_first_word = 0
				#found_valid_second_word = 0
				#k = 0
				#array = [ 0 for x in range(2*number_of_waveform_words_in_a_waveform_packet) ]
				mode = "looking_for_start_of_header"
		if number_of_waveform_words_in_a_waveform_packet==k:
			#did_this += 1
			#found_valid_first_word = 0
			#found_valid_second_word = 0
			#k = 0
			#debug2(debug2_string)
			#debug2_string = ""
			mode = "looking_for_start_of_header"
			#info("on the " + str(waveformfile_waveform_number) + "th waveform in the file")
			yield (carrier_event_number_lsbs, carrier, asic, channel, physical_window, logical_window, n_windows, starting_sample, n_samples, array)
			k = 0
			array = [ 0 for x in range(2*number_of_waveform_words_in_a_waveform_packet) ]
	#info("k=" + str(k))
	#info("first=" + str(first))
	#info("second=" + str(second))
	#info("yielding=" + str(yielding))
	#info("did_this=" + str(did_this))
	if be_verbose_when_collecting_data:
		if len(debug_string):
			debug(debug_string)
#	if len(array):
#		#info("len(array)=" + str(len(array)))
#		info("on the " + str(waveformfile_waveform_number) + "th waveform in the file")
	#yield (carrier_event_number_lsbs, carrier, asic, channel, physical_window, logical_window, n_windows, starting_sample, n_samples, array)
	#debug2(debug2_string)
	#info("words=" + str(words))
	#set_verbosity(old_verbosity)

def events_from_waveformfile():
	event = []
	array = []
	previous_carrier_event_number_lsbs = 123456789 # dummy much bigger than a real carrier event number
	if be_verbose_when_collecting_data:
		info("evnt ce phy log w st sa")
	#old_verbosity = set_verbosity(5)
	global waveformfile_event_number
	for waveform in waveforms_from_waveformfile():
		carrier_event_number_lsbs, carrier, asic, channel, physical_window, logical_window, n_windows, starting_sample, n_samples, array = waveform
		if previous_carrier_event_number_lsbs!=carrier_event_number_lsbs:
			previous_carrier_event_number_lsbs = carrier_event_number_lsbs
			waveformfile_event_number += 1
			if be_verbose_when_collecting_data:
				info(hex(waveformfile_event_number,4) + " " + hex(carrier_event_number_lsbs, 2) + " " + hex(physical_window, 3) + " " + hex(logical_window, 3) + " " + hex(n_windows, 1) + " " + hex(starting_sample, 2) + " " + hex(n_samples, 2))
			if len(event):
				#info("on the " + str(waveformfile_event_number) + "th event in the file")
				yield event
				event = []
		if be_verbose_when_collecting_data:
			debug(hex(waveformfile_event_number,4) + " " + hex(carrier_event_number_lsbs, 2) + " " + hex(carrier, 1) + " " + hex(asic, 1) + " " + hex(channel, 1) + " " + hex(physical_window, 3) + " " + hex(logical_window, 3) + " " + hex(n_windows, 1) + " " + hex(starting_sample, 2) + " " + hex(n_samples, 2))
		event.append(waveform)
	if len(event):
		#info("on the " + str(waveformfile_event_number) + "th event in the file")
		yield event
	#set_verbosity(old_verbosity)

def count_events_in_waveformfile():
	k = 0
	open_waveformfile_for_reading()
	for event in events_from_waveformfile():
		k += 1
	close_waveformfile_for_reading()
	return k

# sudo apt install -y python-pip python3-pip python-tk
# python -m pip install -U pip
# python -m pip install --user -U matplotlib
# python -m pip install --user -U scipy
import matplotlib.pyplot as plt
#import matplotlib.image as mpl_img
from matplotlib.ticker import (MultipleLocator, FormatStrFormatter, AutoMinorLocator)
import numpy
import Tkinter
from PIL import Image, ImageTk # sudo apt-get install python-imaging-tk

gui_update_period_ms = 200

def update_gui():
	if not gui_initialized:
		error("gui not initialized")
		sys.exit(1)
	#info("updating")
	show_latest_png_if_necessary()
	gui_window.after(gui_update_period_ms, update_gui)

plot_width = 1500
plot_height = 700
dpi = 100

gui_initialized = 0
def initialize_gui():
	global gui_window
	global image_frame
	global gui_initialized
	if not gui_initialized:
		gui_initialized = 1
		gui_window = Tkinter.Tk()
		gui_window.title("XRM")
	#infotext = Tkinter.Label(gui_window, text="xrm!")
	#infotext.pack()
	#quitbutton = Tkinter.Button(gui_window, text="quit", command=gui_window.quit)
	#quitbutton.pack()
	image_frame = Tkinter.Frame(gui_window, width=plot_width, height=plot_height)
	image_frame.pack()
	gui_window.after(gui_update_period_ms, update_gui)
	gui_window.mainloop()

def show_png(filename):
	#gui_window.geometry('+%d+%d' % (100,100))
	#gui_window.geometry('%dx%d' % (image1.size[0],image1.size[1]))
	#label_image.place(x=0,y=0,width=image1.size[0],height=image1.size[1])
	try:
		photo = ImageTk.PhotoImage(Image.open(filename))
	except:
		return
	global first_time_through
	global label
	try:
		if first_time_through:
			first_time_through = 0
	except:
		first_time_through = 0
		label = Tkinter.Label(image_frame, image=photo)
		label.pack()
	label.configure(image=photo)
	label.image = photo
	gui_window.update()

def get_latest_png():
	command = "find data -type f -name '*.png' -printf '%TY-%Tm-%Td+%TH:%TM:%TS %p\n' | sort -n -k 1 | tail -n1 | awk '{ print $2 }'"
	response = commands.getstatusoutput(command)
	return response[1]

def show_latest_png():
	filename = get_latest_png()
	show_png(filename)

old_png_filename = ""
def show_latest_png_if_necessary():
	global old_png_filename
	filename = get_latest_png()
	if old_png_filename!=filename:
		old_png_filename = filename
		time.sleep(1)
		info(filename)
		show_png(filename)

def continuously_show_latest_png():
	initialize_gui()

duration_in_ns_of_64_sample_waveform = 23.58
desired_x_axis = "time"

def xrm_map_linear_to_cac(element):
	if element not in range(1, 1+128):
		error("called xrm_map_linear_to_cac(" + str(element) + ") which is not allowed")
		sys.exit(1)
#	string = "element" + dec(element, 3)
	element -= 1
	carrier = element%4
	element -= carrier
	asic_l = element%8
	element -= asic_l
	asic_l /= 4
	channel = element%64
	element -= channel
	channel /= 8
	channel = 7 - channel
	asic_h = element
	asic_h /= 64
	asic_h <<= 1
	asic = asic_h | asic_l
#	string += " carrier" + str(carrier)
#	string += " asic" + str(asic)
#	string += " channel" + str(channel)
#	if not carrier in range(4):
#		error("bad mapping: carrier cannot be " + str(carrier))
#		info(string)
#		sys.exit(0)
#	if not asic in range(4):
#		error("bad mapping: asic cannot be " + str(asic))
#		info(string)
#		sys.exit(0)
#	if not channel in range(8):
#		error("bad mapping: channel cannot be " + str(channel))
#		info(string)
#		sys.exit(0)
	return carrier, asic, channel

def xrm_map_cac_to_linear(carrier, asic, channel):
	#return 32*carrier + 8*asic + channel # straight cac mapping
	#return 16*carrier + 8*asic + channel # straight cac mapping; only carrier0 and carrier2 populated
	#return 1 + carrier + 4*(asic%2) + 64*(asic>>1) + 8*(7-channel) # xrm slac detector wirebond board mapping
	return 1 + carrier + ((asic%2)<<2) + ((asic/2)<<6) + ((7-channel)<<3) # xrm slac detector wirebond board mapping

def show_xrm_mapping():
	for element in range(1, 1+128):
	#for element in range(128):
		carrier, asic, channel = xrm_map_linear_to_cac(element)
#		element += 1
	for carrier in range(4):
		for asic in range(4):
			element_string = ""
			for channel in range(8):
				element = xrm_map_cac_to_linear(carrier, asic, channel)
				element_string += dec(element, 3) + " "
			info(element_string)
	for element_in in range(1, 1+128):
		carrier, asic, channel = xrm_map_linear_to_cac(element_in)
		element = xrm_map_cac_to_linear(carrier, asic, channel)
		if element!= element_in:
			error("bad mapping")

def plotquit(event):
	#info("quitting plots")
	global should_keep_going
	should_keep_going = 0
	global sweep_should_keep_going
	sweep_should_keep_going = 0
	#sys.exit(0)

def cac_phy_sub_sample_array(value):
	return [[[[[[value for sample in range(32)] for subwindow in range(2)] for physical_window in range(512)] for channel in range(8)] for asic in range(4)] for carrier in range(4)]

def lin_phy_sub_sample_array(value):
	return [[[[value for sample in range(32)] for subwindow in range(2)] for physical_window in range(512)] for channel in range(128)]

def lin_array(value):
	#return [[value for sample in range(64*512)] for channel in range(128)]
	return [[value for sample in range(64*number_of_physical_windows_to_plot)] for channel in range(128)]

#independent_axis_units = duration_in_ns_of_64_sample_waveform
independent_axis_units = 12

def lin_ch_domain(first_physical_window_to_plot, last_physical_window_to_plot):
	#return numpy.linspace(first_physical_window_to_plot*duration_in_ns_of_64_sample_waveform, (last_physical_window_to_plot-first_physical_window_to_plot+1)*duration_in_ns_of_64_sample_waveform, (last_physical_window_to_plot-first_physical_window_to_plot+1)*64) # in ns
	return numpy.linspace(first_physical_window_to_plot*independent_axis_units, (last_physical_window_to_plot+1)*independent_axis_units, number_of_physical_windows_to_plot*64) # in ns

#def domain(number_of_physical_windows_to_plot=1):
#	return numpy.linspace(0+duration_in_ns_of_64_sample_waveform*(number_of_physical_windows_to_plot-1), number_of_physical_windows_to_plot*duration_in_ns_of_64_sample_waveform, 64) # in ns

datafile = None
waveformfile = None

pedestal_subtraction_amount = 1000
stagger_amount = 50
ch_adc_min = -300
ch_adc_max = 6800
maximum_absolute_value_before_issuing_warning = 1000

already_plotted = 0
colorarray = [ "red", "green", "orange", "purple", "blue", "saddlebrown", "dodgerblue", "black" ]
def plot_waveforms_staggered_from_waveformfile(desired_event_numbers):
	parsing_and_plotting_start = mytimer()
	if 1==len(desired_event_numbers) and -1==desired_event_numbers[0]:
		N = count_events_in_waveformfile()
		desired_event_numbers = [ n for n in range(N) ]
	number_of_plots = len(desired_event_numbers)
	if 0==number_of_plots:
		return
	global already_plotted
	ch_stagger_offset = numpy.linspace(stagger_amount, stagger_amount, 32)
	if not already_plotted:
		global fig
		fig = plt.figure(figsize=(plot_width/dpi, plot_height/dpi))
		global axes1
		axes1 = fig.add_subplot(1, 1, 1)
		#axes2 = fig.add_subplot(1, 2, 2)
		fig.tight_layout(pad=2)
	if already_plotted:
		axes1.clear()
	if "time"==desired_x_axis:
		axes1.set_ylabel("channel/amplitude combination (arb. units)")
		axes1.set_autoscaley_on(False)
		if use_pedestal_subtraction_mode:
			axes1.set_ylim([ch_adc_min, ch_adc_max])
		else:
			axes1.set_ylim([ch_adc_min+pedestal_subtraction_amount, ch_adc_max+pedestal_subtraction_amount])
		axes1.set_yticklabels([])
		axes1.set_yticks([])
		#axes1.set_xlabel("time (ns)")
		axes1.set_xlabel("RF bucket #")
#		xticksarray = []
#		if number_of_plots<5:
#			for i in range(13*number_of_plots):
#				x = i/0.5088875
#			#xticksarray.append(int(0.5+x*100.0)/100.0)
#				xticksarray.append(x)
#			#axes1.axvline(x=x, linewidth=1, color="grey")
#		else:
#			for i in range(number_of_plots+1):
#				x = i*RF_buckets_per_read_window/0.5088875
#				xticksarray.append(x)
#		axes1.set_xticks(xticksarray)
		axes1.set_autoscalex_on(False)
		if 10<number_of_plots:
			xlim_extra = 50
			#axes1.set_xlim([0-100, 10062+100])
			#axes1.xaxis.set_major_locator(MultipleLocator(508.8875))
			#axes1.xaxis.set_minor_locator(MultipleLocator(42.407/2.0))
			axes1.xaxis.set_major_locator(MultipleLocator(512))
			axes1.xaxis.set_minor_locator(MultipleLocator(12))
			axes1.grid(axis='x', which='minor')
		else:
			xlim_extra = 0.25
			#axes1.xaxis.set_major_locator(MultipleLocator(period_of_RF_clock_ns))
			axes1.xaxis.set_major_locator(MultipleLocator(independent_axis_units/12))
			#axes1.xaxis.set_minor_locator(MultipleLocator())
			axes1.grid(axis='x', which='major')
	else:
		axes1.set_xlabel("channel/amplitude combination (arb. units)")
		axes1.set_autoscalex_on(False)
		axes1.set_xlim([ch_adc_min, ch_adc_max])
		axes1.set_xticklabels([])
		axes1.set_xticks([])
		axes1.set_ylabel("time (ns)")
		if number_of_plots<5:
			yticksarray = []
			for i in range(13*number_of_plots):
				y = i/0.5088875
				#yticksarray.append(int(0.5+y*100.0)/100.0)
				yticksarray.append(y)
				#axes1.axhline(y=y, linewidth=1, color="grey")
			axes1.grid(axis='y', which='major')
			axes1.set_yticks(yticksarray)
	#old_verbosity = set_verbosity(5)
	array = []
	i = 0 # i is the number of the waveform in the file
	#j = 0 # j is the number of waveforms we're plotting
	k = 0 # k is the number of events we've found in the file
	m = 0 # m is the number of events we've added to the plot
	#l = 0 # l is the number of the sample in the 32 sample waveform
	event_number_string = ""
	#data = lin_phy_sub_sample_array(0)
	data = lin_array(0)
	found_an_event_to_plot = 0
	global waveformfile_event_number
	first_physical_window_seen = 512
	last_physical_window_seen  = 0
	#counter = 0
	for event in events_from_waveformfile():
		#if :
		#	k = waveformfile_event_number - 1
		#else:
		#	k += 1
		#info("waveformfile_event_number=" + str(waveformfile_event_number))
		for desired_event_number in desired_event_numbers:
			#info("den:waveformfile_event_number " + str(desired_event_number) + ":" + str(waveformfile_event_number))
			# desired_event_number==0 means the last one in the file
			if 1==len(desired_event_numbers) and 0==desired_event_number:
				if waveformfile_event_number:
					desired_event_number = waveformfile_event_number
				else:
					desired_event_number = 1
#				N = count_events_in_waveformfile()
#				if be_verbose_when_collecting_data:
#					info("total events in file: " + str(N))
#				desired_event_number = N - 1
#				if 0==desired_event_number:
#					desired_event_number = N
#			if 0==N:
#				continue
			#info("den:waveformfile_event_number " + str(desired_event_number) + ":" + str(waveformfile_event_number))
			if desired_event_number==waveformfile_event_number:
				found_an_event_to_plot = 1
				event_number_string = dec(desired_event_number, 4)
				m += 1
				if 1<len(desired_event_numbers):
					axes1.set_title("\"" + waveformfilename + "\"")
				else:
					axes1.set_title("event #" + dec(desired_event_number) + " from \"" + waveformfilename + "\"")
				#seen_in_this_event = create_cac_matrix(0)
				for waveform in event:
					carrier_event_number_lsbs, carrier, asic, channel, physical_window, logical_window, n_windows, starting_sample, n_samples, array = waveform
					first_physical_window_seen = min(first_physical_window_seen, physical_window)
					last_physical_window_seen  = max(last_physical_window_seen,  physical_window)
					#counter += 1
					#if counter<20:
					#	info(hex(physical_window, 3) + "[" + hex(first_physical_window_seen,3) + "," + hex(last_physical_window_seen,3) + "]")
#					max_absolute_value_seen = 0
					fudge = 0
					#if 2==carrier and 3==asic and 7==channel:
					#	fudge = 1
#					if 0==carrier and 0==asic and 0==channel:
#						info(hex(waveformfile_event_number,4) + " " + hex(carrier_event_number_lsbs, 2) + " " + hex(carrier, 1) + " " + hex(asic, 1) + " " + hex(channel, 1) + " " + hex(physical_window, 3) + " " + hex(logical_window, 3) + " " + hex(n_windows, 1) + " " + hex(starting_sample, 2) + " " + hex(n_samples, 2))
					if not fudge:
						#seen_in_this_event[carrier][asic][channel] += 1
						lin_ch = xrm_map_cac_to_linear(carrier, asic, channel)
						if 1==number_of_physical_windows_to_plot:
							sample = starting_sample
						else:
							sample = 64*physical_window + starting_sample
						#info(str(lin_ch) + " " + str(sample))
						data[lin_ch][sample:sample+32] = array + lin_ch * ch_stagger_offset
#					p = 0
#					for value in array:
#						if maximum_absolute_value_before_issuing_warning<abs(value):
#							fudge += 1
	#						if 2<p:
	#							array[p] = array[p-2] # fudge!
	#						else:
	#							array[p] = 0 # fudge!
#						p += 1
#						max_absolute_value_seen = max(max_absolute_value_seen, abs(value))
						#maximum_absolute_value_before_issuing_warning<max_absolute_value_seen:
						#info(hex(waveformfile_event_number,4) + " " + hex(carrier_event_number_lsbs, 2) + " " + hex(carrier, 1) + " " + hex(asic, 1) + " " + hex(channel, 1) + " " + hex(physical_window, 3) + " " + hex(logical_window, 3) + " " + hex(n_windows, 1) + " " + hex(starting_sample, 2) + " " + hex(n_samples, 2))
						#warning("max absolute value seen: " + hex(max_absolute_value_seen))
						#info(hex(carrier, 1) + " " + hex(asic, 1) + " " + hex(channel, 1) + " " + str(array))
					i += 1
				if be_verbose_when_collecting_data:
					info("waveforms seen in this event: " + str(i))
					#show_cac_matrix(seen_in_this_event, 1)
					#info("waveforms to plot: " + str(j))
				#fig.canvas.draw_idle()
		if be_verbose_when_collecting_data:
			info("number of events seen in file: " + str(waveformfile_event_number))
	#plotting_start = mytimer()
	if found_an_event_to_plot:
		#info("first_physical_window_seen=" + hex(first_physical_window_seen,3))
		#info(" last_physical_window_seen=" + hex(last_physical_window_seen,3))
#		if last_physical_window_seen<first_physical_window_seen:
#			error("last<first")
		if last_physical_window_seen-first_physical_window_seen+1!=number_of_physical_windows_to_plot:
			error("tot wrong")
			last_physical_window_seen = number_of_physical_windows_to_plot - first_physical_window_seen - 1
		axes1.set_xlim([first_physical_window_seen*independent_axis_units-xlim_extra, (last_physical_window_seen+1)*independent_axis_units+xlim_extra])
		for lin_ch in range(128):
			if lin_ch%2:
				if "time"==desired_x_axis:
					#axes1.plot(lin_ch_domain(0, number_of_physical_windows_to_plot-1), data[lin_ch], color=colorarray[lin_ch%8])
					axes1.plot(lin_ch_domain(first_physical_window_seen, last_physical_window_seen), data[lin_ch], color=colorarray[lin_ch%8])
		if 1<len(desired_event_numbers):
			png_filename = waveformfilename + "." + str(m) + "events.png"
		else:
			png_filename = waveformfilename + "." + event_number_string + ".png"
		#info("plotted event number " + str(event_number_string))
		fig.savefig(png_filename)
		set_mtime(png_filename, waveformfilename)
		info("wrote file " + png_filename)
		if not already_plotted:
			already_plotted = 1
			if show_plot:
				plt.ion()
				plt.show()
				plt.connect("close_event", plotquit)
		if show_plot:
			fig.canvas.draw()
			fig.canvas.flush_events()
			#plt.draw()
			#plt.pause(0.01)
			#plt.ioff()
		#show_mytimer(plotting_start, "plotting")
		#show_mytimer(parsing_and_plotting_start, "parsing and plotting")

def plot_waveforms_from_waveformfile(desired_event_numbers):
	plot_waveforms_staggered_from_waveformfile(desired_event_numbers)
	#plot_waveforms_overlayed_from_waveformfile(desired_event_numbers)

def threaded_xrm_trigger_boardstack_and_capture_data(N, duration):
	if be_verbose_when_collecting_data:
		info("expecting " + str(N) + " events")
	mythread = threading.Thread(target=ethudp.capture_triggered_data, args=(datafilename, N, number_of_bytes_to_expect_per_event, be_verbose_when_collecting_data))
	mythread.start()
	diff = trigger_and_wait_for_xrm_triggers(duration)
	#diff = allow_xrm_triggers_for_a_while(duration)
	mythread.join()
	if not diff == N:
		warning("discrepancy: " + str(diff) + " (event # difference) != " + str(N) + " (desired number of triggers)")

should_software_trigger = 0
extra_duration_to_wait_for_data = 0.001
def get_triggered_data(N, n, N_log2, desired_event_numbers):
	# N is how many sets of events that we want
	# n is how many bunch markers are active per set (usually from a call to count_frame_masks())
	# plot_continuous==1 means plot each one right after acquisition
	set_desired_trigger_quantities(N, N, N, N)
	set_trig_prescale_N_log2(N_log2)
	enable_xrm_triggers()
	wait_at_least_until_next_frame9()
	duration = ((N*n)+0.0) / expected_trigger_rate[N_log2] + extra_duration_to_wait_for_data
	first_time_through = 1
	global should_keep_going
	should_keep_going = 1
	while should_keep_going:
		manualRunReset_if_necessary(BS)
		threaded_xrm_trigger_boardstack_and_capture_data(N*n, duration)
		if be_verbose_when_collecting_data:
			show_some_info()
		if len(desired_event_numbers):
			quick_temp_status(BS)
			if plot_continuous:
				parse_datafile_and_dump_waveforms_to(waveformfilename)
				plot_waveforms_from_waveformfile(desired_event_numbers)
			if "acquire"==xrm_mode:
				read_config_file_for_xrm_stuff()
				get_ready_for_data(BS)
				cycle_calibration_channel(BS)
				wait_for_PS_on_each_carrier_to_pick_up_the_changes(BS)
#		if not xrm_run_continuous:
#			info("mode changed (from configfile) to no longer run in continuous mode")
#			if already_plotted:
#				plt.close("all")
#			should_keep_going = 0
#			break
		if datafilesize<file_size(datafilename):
			generate_new_datafilename()
		if first_time_through:
			first_time_through = 0
			if "acquire"==xrm_mode:
				should_keep_going = xrm_run_continuous
			if "sweep"==xrm_mode:
				should_keep_going = 0
#		if plot_continuous and not should_keep_going:
#			plt.pause(0)
		if should_keep_going and "acquire"==xrm_mode:
			if should_cycle_slowly_through_bunch_markers:
				cycle_through_bunch_marker_a()
			time.sleep(desired_pause_between_acquisitions)
			info("")
	disable_xrm_triggers()

def sweep_bunch_marker_positions():
	set_bunch_marker(0, 0b100000000, bunch_marker_first_valid)
	set_bunch_marker(1, 0b000000000, 0)
	set_bunch_marker(2, 0b000000000, 0)
	set_bunch_marker(3, 0b000000001, bunch_marker_last_valid)
	set_trig_prescale_N_log2(2)
	start_frame = 8
	end_frame = 0
	#info("sweep")
	for j in range(start_frame, end_frame-1, -1):
		for i in range(bunch_marker_first_valid, bunch_marker_last_valid+1):
			#info(str(j) + " " + str(i))
			set_bunch_marker(1, 1<<j, i)
			set_bunch_marker(2, 1<<(8-j), bunch_marker_last_valid + bunch_marker_first_valid - i)
		#time.sleep(0.1)
		wait_at_least_until_next_frame9()

def clear_bunch_markers():
	set_bunch_marker(0, 0, 0)
	set_bunch_marker(1, 0, 0)
	set_bunch_marker(2, 0, 0)
	set_bunch_marker(3, 0, 0)

def cycle_calibration_channel(BS, init=0):
	global calibration_channel_state
	if init or 128<calibration_channel_state or calibration_channel_state<1:
		calibration_channel_state = 1
	carrier, asic, channel = xrm_map_linear_to_cac(calibration_channel_state)
	select_calibration_channel_single_destination(BS, carrier, asic, channel)
	calibration_channel_state += 2
	return calibration_channel_state

def cycle_through_bunch_marker_a():
	frame_mask_a = 0b100000000
	global bunch_marker_a
	try:
		bunch_marker_a += int(RF_buckets_per_read_window)
		if bunch_marker_last_valid<bunch_marker_a:
			bunch_marker_a = bunch_marker_first_valid
	except:
		bunch_marker_a = initial_bunch_marker_of_interest
	set_bunch_marker(0, frame_mask_a, bunch_marker_a)
	wait_at_least_until_next_frame9()

def sweep_bunch_marker_positions_and_take_data_at_each_position():
	frame_mask_a = 0b100000000
	frame_mask_b = 0b000000001
	frame_mask_c = 0b000010000
	frame_mask_d = 0b000000100
	how_many_bunch_markers_to_activate = 1
	step = int(RF_buckets_per_read_window)
	fullstep = how_many_bunch_markers_to_activate * step
	first = bunch_marker_first_valid
	last = bunch_marker_last_valid - step
	first_time_through = 1
	global sweep_should_keep_going
	sweep_should_keep_going = 1
	clear_bunch_markers()
	while sweep_should_keep_going:
		calch = cycle_calibration_channel(BS, 1)
		wait_for_PS_on_each_carrier_to_pick_up_the_changes(BS)
		start = mytimer()
		for i in range(first, last, fullstep):
			info("setting bunch_marker_a to RF_bucket # " + str(i), 0)
			set_bunch_marker(0, frame_mask_a, i)
#			if 1<how_many_bunch_markers_to_activate:
#				info("setting bunch_marker_b to RF_bucket # " + str(i+1*step), 0)
#				set_bunch_marker(1, frame_mask_b, i+1*step)
#			if 2<how_many_bunch_markers_to_activate:
#				info("setting bunch_marker_c to RF_bucket # " + str(i+2*step), 0)
#				set_bunch_marker(2, frame_mask_c, i+2*step)
#			if 3<how_many_bunch_markers_to_activate:
#				info("setting bunch_marker_d to RF_bucket # " + str(i+3*step), 0)
#				set_bunch_marker(3, frame_mask_d, i+3*step)
			wait_at_least_until_next_frame9()
			if sweep_should_keep_going:
				desired_event_numbers = []
				#if 1300==i or 2596==i or 3892==i or 5104==i:
				if 5104==i:
					desired_event_numbers = [x for x in range(1, int(i/step) + 1)]
					#info(str(desired_event_numbers))
				#start = mytimer()
				get_triggered_data(1, count_frame_masks(), trig_prescale_N_log2, desired_event_numbers)
				#show_mytimer(start)
				if calch<129:
					calch = cycle_calibration_channel(BS)
					wait_for_PS_on_each_carrier_to_pick_up_the_changes(BS)
				elif 129==calch:
					calch = 500
					disable_calibration_input(BS)
					wait_for_PS_on_each_carrier_to_pick_up_the_changes(BS)
			else:
				return
		show_mytimer(start, "sweep of entire revolution")
		if xrm_run_continuous:
			#get_status(BS)
			read_config_file_for_xrm_stuff()
			generate_new_datafilename()
			time.sleep(desired_pause_between_acquisitions)
			info("")
		if first_time_through:
			first_time_through = 0
			sweep_should_keep_going = xrm_run_continuous
	disable_calibration_input(BS)

def threshold_tweak(BS):
	#show_scalers(BS)
	#info("")
#	for carrier in range(2, 3):
#		for asic in range(2,4):
#			for channel in range(4,8):
#				tweak_asic_trigger_comparator_threshold(BS, carrier, asic, channel, -5)
#				time.sleep(0.01)
	scrod_serial = get_scrod_serial(BS)
	#info(str(scrod_serial))
	if 108==scrod_serial:
		tweak_asic_trigger_comparator_threshold(BS, 2, 3, 7, -6)
		tweak_asic_trigger_comparator_threshold(BS, 0, 2, 7, -5)
	if 4==scrod_serial:
		tweak_asic_trigger_comparator_threshold(BS, 0, 0, 0, -24)
		tweak_asic_trigger_comparator_threshold(BS, 0, 0, 1, -24)
		tweak_asic_trigger_comparator_threshold(BS, 0, 1, 0, -6)
		tweak_asic_trigger_comparator_threshold(BS, 0, 1, 1, -6)
		tweak_asic_trigger_comparator_threshold(BS, 0, 3, 0, -8)
		tweak_asic_trigger_comparator_threshold(BS, 0, 3, 1, -8)
		tweak_asic_trigger_comparator_threshold(BS, 2, 0, 0, -6)
		tweak_asic_trigger_comparator_threshold(BS, 2, 0, 1, -6)
		tweak_asic_trigger_comparator_threshold(BS, 2, 1, 0, -24)
		tweak_asic_trigger_comparator_threshold(BS, 2, 1, 1, -24)
		tweak_asic_trigger_comparator_threshold(BS, 2, 2, 0, -6)
		tweak_asic_trigger_comparator_threshold(BS, 2, 2, 1, -6)
		tweak_asic_trigger_comparator_threshold(BS, 2, 3, 0, -6)
		tweak_asic_trigger_comparator_threshold(BS, 2, 3, 1, -6)
	#show_scalers(BS)

def setup_xrm_trigger():
	manualRunReset(BS)
	#set_trig_prescale_N_log2(14)
	clear_bunch_markers()
	set_bunch_marker(0, 0b100000000, initial_bunch_marker_of_interest)
	#set_bunch_marker(0, 0b100000000, bunch_marker_first_valid)
	if be_verbose_when_collecting_data:
		show_relevant_registers()
	cycle_calibration_channel(BS, 1)
#	set_bunch_markers(1000, 2000, 3000, 4000)
#	set_quadbunch_markers(100, 102, 104, 106)
#	set_quadbunch_markers(100, 100, 100, 100)
#	set_asic_write_window_bunch_markers(1, 3, 5, 7)
#	set_asic_write_window_bunch_markers(1, 1, 1, 1)
#	set_quadbunch_markers(2,4,6,8)

def get_ready_for_data(BS):
	if be_verbose_when_collecting_data:
		info("getting ready for data")
	# CARRIER_IRSX_readoutWindows=1 SCROD_PS_maxRawWaveforms=64 total_frame_masks_active=3 (consecutive); numchunks=6
	# CARRIER_IRSX_readoutWindows=1 SCROD_PS_maxRawWaveforms=128 total_frame_masks_active=3 (consecutive); numchunks=9
	# ...phaseRead loops from prep1asic_data_trueROI or the PS code...
#	write_register_for_each_carrier(CARRIER_PS_cal7minusCh, 7, BS) # this means ch 0
#	#write_register_for_each_carrier(CARRIER_PS_cal7minusCh, 0, BS) # this means ch 7
#	write_register(CARRIER_PS_calEn, 0x0, BS, 0)
#	write_register(CARRIER_PS_calEn, 0x0, BS, 1)
#	write_register(CARRIER_PS_calEn, 0x8, BS, 2)
#	write_register(CARRIER_PS_calEn, 0x0, BS, 3)
	write_register(SCROD_PS_MAX_EVENT_QUEUE_DEPTH, 250, BS)
	if be_verbose_when_collecting_data:
		show_register_by_name("SCROD_PS_MAX_EVENT_QUEUE_DEPTH", BS)
	#write_register(SCROD_PS_featureExtMode, featExtractMode_normal, BS)
	write_register(SCROD_PS_featureExtMode, featExtractMode_nocuts, BS)
	#write_register(SCROD_PS_featureExtMode, featExtractMode_passthru, BS)
	if use_pedestal_subtraction_mode:
		turn_on_pedestal_subtraction_mode(BS)
	else:
		turn_off_pedestal_subtraction_mode(BS)
	wait_for_all_5_PS_to_pick_up_the_changes(BS)
	#wait_for_PS_on_SCROD_to_pick_up_the_changes(BS)
	#wait_for_PS_on_each_carrier_to_pick_up_the_changes(BS)
	write_register_for_each_asic(CARRIER_IRSX_convertResetWait, 0x0f, BS)
	write_register_for_each_asic(CARRIER_IRSX_enableTestPattern, 0, BS)
	write_register(SCROD_AxiCommon_filteringTimeout, 127221875, BS)
	write_register_for_each_asic(CARRIER_IRSX_wrAddrFixed, 0x40, BS)     
	#write_register_for_each_asic(CARRIER_IRSX_wrAddrMode, wrAddrMode_Calibration, BS)
	write_register_for_each_asic(CARRIER_IRSX_wrAddrMode, wrAddrMode_Cyclic, BS)
	write_register_for_each_asic(CARRIER_IRSX_readoutMode, readoutMode_Forced, BS)
	#write_register_for_each_asic(CARRIER_IRSX_readoutMode, readoutMode_Lookback, BS)
	#write_register(SCROD_AxiCommon_oserdes_word_trig, 0, BS)
	#write_register(SCROD_AxiCommon_oserdes_word_trig, (0x55<<16) | 0x00, BS)
	#write_register(SCROD_AxiCommon_oserdes_word_trig, (0x00<<16) | 0xaa, BS)
	#write_register(SCROD_AxiCommon_oserdes_word_trig, (0x00<<16) | 0xe7, BS)
	write_register(SCROD_AxiCommon_oserdes_word_trig, (0xaa<<16) | 0xaa, BS)
	write_register_for_each_asic(CARRIER_IRSX_wrAddrStart, 0x00, BS)
	write_register_for_each_asic(CARRIER_IRSX_wrAddrStop,  0xe0, BS)
	#write_register_for_each_asic(CARRIER_IRSX_wrAddrStop,  0xff, BS)
	write_register_for_each_asic(CARRIER_IRSX_wrAddrJunk,  0xff, BS)
	write_register_for_each_asic(CARRIER_IRSX_writesAfterTrig, 0x00, BS)
	#write_register_for_each_asic(CARRIER_IRSX_readoutLookback, 0x000, BS) # set by set_bunch_marker()
	write_register_for_each_asic(CARRIER_IRSX_trigAdj, 0, BS)
	unmask_triggers(0xa, 0x0, 0xf, 0x0, 0xf)
	write_register(SCROD_AxiCommon_asicMask, 0xf0f0, BS)
	#time.sleep(4.0)

def setup_xrm_mode(BS):
	set_verbosity_of_virtual_tty(BS, 3)
	setup_xrm_trigger()
	set_nominal_asic_trigger_comparator_thresholds(BS)
	number_of_bytes_per_word = 4
	number_of_bytes_for_event_header = 5 * number_of_bytes_per_word
	number_of_enabled_channels = 64
	desired_number_of_readout_windows_per_channel = 1 # 64 samples per window
	number_of_waveforms_per_channel = 2 * desired_number_of_readout_windows_per_channel # 32 samples per waveform
	number_of_waveforms_to_expect_from_forced_mode_with_our_channel_masks = number_of_waveforms_per_channel * number_of_enabled_channels
	number_of_bytes_for_hits = number_of_waveforms_to_expect_from_forced_mode_with_our_channel_masks * 5 * number_of_bytes_per_word
	expected_number_of_bytes_for_waveforms = number_of_waveforms_to_expect_from_forced_mode_with_our_channel_masks * 18 * number_of_bytes_per_word
	global number_of_bytes_to_expect_per_event
	number_of_bytes_to_expect_per_event = number_of_bytes_for_event_header + number_of_bytes_for_hits + expected_number_of_bytes_for_waveforms
	if be_verbose_when_collecting_data:
		info("number_of_bytes_to_expect_per_event: " + str(number_of_bytes_to_expect_per_event))
	write_register_for_each_asic(CARRIER_IRSX_readoutWindows, desired_number_of_readout_windows_per_channel, BS)
	write_register(SCROD_PS_maxRawWaveforms, number_of_waveforms_to_expect_from_forced_mode_with_our_channel_masks, BS)
	#number_of_bytes_to_expect_per_ASDF = count_frame_masks() * number_of_bytes_to_expect_per_event
	#threshold_tweak(BS)
	#set_desired_trigger_quantities(10, 0, 0, 0)
	#show_register_by_name("SCROD_AxiCommon_desired_trigger_quantity_for_bunch_marker_a", BS)
	#show_register_by_name("SCROD_AxiCommon_desired_trigger_quantity_for_bunch_marker_b", BS)
	#show_register_by_name("SCROD_AxiCommon_desired_trigger_quantity_for_bunch_marker_c", BS)
	#show_register_by_name("SCROD_AxiCommon_desired_trigger_quantity_for_bunch_marker_d", BS)

def generate_new_datafilename():
	close_datafile_for_reading()
	close_waveformfile_for_reading()
	timestring = time.strftime("%Y-%m-%d.%H%M%S")
	global datafilename
	datafilename     = "data/" + timestring + ".datafile"
	global waveformfilename
	waveformfilename = "data/" + timestring + ".waveforms"
	info("")
	info("datafile name: " + datafilename)
	if use_pedestal_subtraction_mode and retake_pedestals_at_every_filechange:
		take_pedestals(BS)
		manualRunReset(BS)
	get_ready_for_data(BS)

def read_config_file_for_xrm_stuff():
	global plot_continuous
	plot_continuous = read_integer_from_config_file_or_use_default("plot_continuous", 0)
	global be_verbose_when_collecting_data
	be_verbose_when_collecting_data = read_integer_from_config_file_or_use_default("be_verbose_when_collecting_data", 0)
	global datafilesize
	datafilesize = read_integer_from_config_file_or_use_default("datafilesize", 100000000)
	global xrm_run_continuous
	xrm_run_continuous = read_integer_from_config_file_or_use_default("xrm_run_continuous", 0)
	global retake_pedestals_at_every_filechange
	retake_pedestals_at_every_filechange = read_integer_from_config_file_or_use_default("retake_pedestals_at_every_filechange", 0)
	global xrm_mode
	xrm_mode = read_string_from_config_file_or_use_default("xrm_mode", "acquire")
	global number_of_physical_windows_to_plot
	if "sweep"==xrm_mode:
		number_of_physical_windows_to_plot = 426
	else:
		number_of_physical_windows_to_plot = 1
	#info(xrm_mode)
	global show_plot
	show_plot = read_integer_from_config_file_or_use_default("show_plot", 0)
	global use_pedestal_subtraction_mode
	use_pedestal_subtraction_mode = read_integer_from_config_file_or_use_default("use_pedestal_subtraction_mode", 1)
	global desired_pause_between_acquisitions
	desired_pause_between_acquisitions = read_integer_from_config_file_or_use_default("desired_pause_between_acquisitions", 0)
	global should_cycle_slowly_through_bunch_markers
	should_cycle_slowly_through_bunch_markers = read_integer_from_config_file_or_use_default("should_cycle_slowly_through_bunch_markers", 0)

def xrm_run(BS):
	calculate_parameters(6, 0)
	#clear_configuration_status(BS)
	if not has_configuration_has_been_done_yet_on_all_carriers(BS):
		configure_boardstack(BS, 0) # trueROI
		configure_boardstack(BS, 1) # fast
		take_pedestals(BS)
		#configure_boardstack_then_acquire_and_check_pedestals(BS)
		for carrier in [ 1, 3 ]:
			#info(str(carrier))
#			juice.juice_carrier(BS, carrier, juice.map_c(carrier))
			write_register(CARRIER_PS_ampRegEn, 0, BS, carrier)
			write_register(CARRIER_PS_ampEn, 0, BS, carrier)
			for asic in range(4):
				juice.juice_asic(BS, carrier, asic, "off")
		wait_for_PS_on_each_carrier_to_pick_up_the_changes(BS)
	#info(juice.verify_juice(BS))
	#config1boardstack_fast(BS)
	#prep1boardstack_data_trueROI(BS)
	#show_xrm_mapping()
	#sys.exit(0)
	desired_event_numbers = [ 0 ]
	read_config_file_for_xrm_stuff()
	global trig_prescale_N_log2
	trig_prescale_N_log2 = 1
	N = 1
	if "parse"==xrm_mode:
		#parse_all_datafiles_in_this_dirtree("data", 1)
		parse_all_waveformfiles_in_this_dirtree("data", 0)
		#parse_all_waveformfiles_in_this_dirtree("data/2019-10-29", 0)
		#parse_datafile_and_dump_waveforms_to(waveformfilename)
		#plot_waveforms_from_waveformfile(desired_event_numbers)
	elif "sweep"==xrm_mode or "acquire"==xrm_mode:
		#global number_of_events_to_acquire_at_a_time 
		#number_of_events_to_acquire_at_a_time = N * count_frame_masks()
		setup_xrm_mode(BS)
		generate_new_datafilename()
		#generate_histogram_of_trigger_rate_versus_prescale_N_log2(0, 18)
		# 0 doesn't work reliably (gets ~56 events out of every 100 triggers)
		# 1 doesn't work reliably (gets ~98 events out of every 100 triggers)
		#number_of_events_to_acquire_at_a_time = 250
		#trig_prescale_N_log2 = 2
		# can handle 3 consecutive revolutions (in frame mask) with 256 waveforms out (1 bunch marker active only)
		if "sweep"==xrm_mode:
			sweep_bunch_marker_positions_and_take_data_at_each_position()
		else:
			get_triggered_data(N, count_frame_masks(), trig_prescale_N_log2, desired_event_numbers)
		#sweep_bunch_marker_positions()
		#show_relevant_registers()
		#setup_xrm_trigger()
		#enable_xrm_triggers()
		#show_xrm_trigger_enable_status()
		#set_desired_trigger_quantities(1, 2, 3, 4)

if __name__ == "__main__":
	parse_standard_argv_options_and_check_numcarrier_on_boardstacks()
	run_this_command_on_all_BS_and_log_results_to_file(xrm_run)