Latest

DaqFileIO.cxx

@@ -4,47 +4,92 @@
 // 
 //
 #include "DaqIo.h"
+#include <assert.h>
+
+//
+// Small dummy function to act as the default callback
+//
+static void DefaultErrorFunc(const char *, const char *) {}
+static void DefaultProgressFunc(const char *, int, int) {}
+
+struct TFrameInfo {
+	int ofs;
+	int numChan;
+};
 
 
 CDaqLowLevelIo::CDaqLowLevelIo()
 {
+	m_pFile         = 0;
+	m_UserErrorFunc = DefaultErrorFunc;
+	m_UserProgrFunc = DefaultProgressFunc;
+
+	m_HaveToc    = false;
+	m_FirstFrame = -1;
+	m_LastFrame  = -1;
+	m_EofStatus  = eDAQ_EOF_UNDEFINED;
+	m_TocExists  = false;
 }
 
 CDaqLowLevelIo::~CDaqLowLevelIo()
 {
+	if ( m_pFile )
+		Close();
 }
 
 
-void
-NearDump(FILE* p)
-{
-	printf("\nData dump near error\n");
-	fseek(p, -8, SEEK_CUR);
-	int d;
 
-	for (int i=0; i<8; i++) {
-		size_t n = fread(&d, sizeof(d), 1, p);
-		printf("%-d:  %d (code=%d)\n", ftell(p)-sizeof(d), d, n);
-	}
+int
+CDaqChannelInfo::GetItemSize() const
+{
+	if ( m_Type == 'f' || m_Type == 'i' ) return 4;
+	if ( m_Type == 'd' ) return 8;
+	if ( m_Type == 's' ) return 2;
+	if ( m_Type == 'c' ) return 1;
+	assert(0);
+	return -1;
 }
 
-int 
-DataSize(CDaqChannelInfo& ch)
+
+
+//////////////////////////////////////////////////////////////////////////////////////
+///
+/// This function allows user to specify error and progress callbacks. The 
+/// error callback provides a string explaining the error and the progress 
+/// callback provides actual/total frames processed. The exact meaning of the 
+/// progress function arguments depends on what activity the library is performing 
+/// when it calls it. 
+///
+void
+CDaqLowLevelIo::SetCallbacks(
+	void errf(const char *, const char *),
+	void progf(const char *, int, int)
+	)
 {
-	int size;
+	if ( errf != 0 ) m_UserErrorFunc = errf; else m_UserErrorFunc = DefaultErrorFunc;
+	if ( progf != 0 ) m_UserProgrFunc = progf; else m_UserProgrFunc = DefaultProgressFunc;
+}
 
-	switch ( ch.m_Type ) {
-		case 'f' : 
-		case 'i' : size = 4; break;
 
-		case 'd' : size = 8; break;
+void
+NearDump(FILE* p, int ofs)
+{
+	int origOfs = ftell(p);
 
-		case 's' : size = 2; break;
-		case 'c' : size = 1; break;
-		default : return -1;
-	}
+	printf("\nData dump near unrecoverable read at offset %d\n", ofs);
+	fseek(p, -8*4, SEEK_CUR);
+	int d;
 
-	return size * ch.m_Items;
+	for (int i=0; i<96; i++) {
+		size_t n = fread(&d, sizeof(d), 1, p);
+		if ( n == 1 ) {
+			printf("%-d:  0x%08X %d\n", ftell(p)-sizeof(d), d, d);
+		}
+		else {
+			printf("Read at ofs %-d failed.\n", ftell(p)-sizeof(d));
+		}
+	}
+	fseek(p, origOfs, SEEK_SET);
 }
 
  
@@ -61,6 +106,7 @@ CDaqLowLevelIo::ReadHeader(	FILE *p)
 	unsigned magic;
 
 	if ( fread(&magic, sizeof(magic), 1, p) != 1 ) {
+		m_LastError = "cannot read magic number";
 		return false;
 	}
 	rewind(p);
@@ -68,13 +114,37 @@ CDaqLowLevelIo::ReadHeader(	FILE *p)
 	bool supported = false;
 
 	if  ( magic == DAQ_MAGIC_NUM_VER_2_0 ) {
-		printf("Version 2.0\n");
+		m_Version = "2.0";
+		struct Hdr {
+			unsigned Magic;
+			char     Title[128];
+			char     Date[27];
+			char     Subj[64];
+			char     Run[64];
+			int      NumEntries;
+			unsigned DAQFrequency;
+		} header = { 0 };
+
+		if ( fread(&header, sizeof(header), 1, p) != 1 ) {
+			m_LastError = "cannot read header";
+			return false;
+		}
+		m_Title      = header.Title;
+		m_Date       = header.Date;
+		m_Subj       = header.Subj;
+		m_Run        = header.Run;
+		m_RunInst    = "Unavailable";
+		m_NumEntries = header.NumEntries;
+		m_Frequency  = header.DAQFrequency;
+		supported = true;
 	}
 	else if ( magic == DAQ_MAGIC_NUM_VER_2_1 ) {
 		printf("Version 2.1\n");
+		exit(0);
 	}
 	else if ( magic == DAQ_MAGIC_NUM_VER_2_2 ) {
-		printf("Version 2.2\n");
+		m_Version = "2.2";
+
 		struct Hdr {
 			unsigned Magic;
 			char     Title[120];
@@ -87,6 +157,7 @@ CDaqLowLevelIo::ReadHeader(	FILE *p)
 		} header = { 0 };
 
 		if ( fread(&header, sizeof(header), 1, p) != 1 ) {
+			m_LastError = "cannot read header";
 			return false;
 		}
 		m_Title      = header.Title;
@@ -104,6 +175,7 @@ CDaqLowLevelIo::ReadHeader(	FILE *p)
 		return false;
 	}
 
+	m_LastError = "";
 	return true;
 }
 
@@ -149,96 +221,1373 @@ CDaqLowLevelIo::ReadChannelInfo(FILE *p)
 }
 
 
+//////////////////////////////////////////////////////////////////////////////////
+//
+// Returns true if the file is currently seeked on a frame header else false.
+// Either way, it leaves the file pointer at the same place it found it unless
+// it run into a read error.
+//
+bool
+CDaqLowLevelIo::DetectFrameHeader(
+	int recentFrame)
+{
+	int origOfs = ftell(m_pFile);
+	int header[3];
+
+	if ( fread(header, sizeof(int), 3, m_pFile) != 3 ) return false;
+
+	// we use a large number for the reasonable frame skip since with
+	// small records, a single DAQ buffer can often be as many frames
+	bool reasonableFrame   = header[1] - recentFrame < 200000;
+	bool reasonableChCount = header[2] >= 0 && header[2] < m_NumEntries;
+
+	fseek(m_pFile, origOfs, SEEK_SET);
+
+	if ( header[0] == -1 && reasonableFrame && reasonableChCount ) {
+		return true;
+	}
+	else {
+		return false;
+	}
+}
+
+
+//////////////////////////////////////////////////////////////////////////////////
+//
+// Attempts to re-sync after reading a bad channel definition;  usually, 
+// there are more channel definitions nearby so we can re-sync there
+//
+CDaqLowLevelIo::TResyncOutcome
+CDaqLowLevelIo::ResyncChannel(
+	int		recentFrame,
+	int&	skip)
+{
+	int  data;
+	int  cch1;		// candidate channels
+	int  tries = 0;
+	int  startingOffset = ftell(m_pFile);
+
+	skip = 0;
+	while ( !feof(m_pFile) && tries++ < 30) {
+		int	mostRecentGoodOfs;
+
+		mostRecentGoodOfs = ftell(m_pFile);
+		if ( fread(&data, sizeof(int), 1, m_pFile) != 1 ) return eRESYNC_NONE;
+
+		// possible channel definition; read at least two more things to be sure
+		// we are back in sync
+		if ( data >=0 && data < m_NumEntries ) {
+			cch1 = data;
+			CDaqChannelInfo& ch = m_Channels[cch1];
+			int  skipDataPortion = ch.GetRecSize();
+
+			fseek(m_pFile, skipDataPortion, SEEK_CUR);
+
+			if ( fread(&data, sizeof(int), 1, m_pFile) != 1 ) return eRESYNC_NONE;
+
+			if ( data >=0 && data < m_NumEntries && data > cch1 ) {
+				// it seems as if we are back in sync, we re-wind and return
+				while ( data >=0 && data < m_NumEntries && data > cch1 ) {
+					CDaqChannelInfo& ch = m_Channels[data];
+					int  skipDataPortion = ch.GetRecSize();
+
+					fseek(m_pFile, skipDataPortion, SEEK_CUR);
+					if ( fread(&data, sizeof(int), 1, m_pFile) != 1 ) return eRESYNC_NONE;
+				}
+			}
+
+			if ( data == -1 ) {
+				fseek(m_pFile, -4, SEEK_CUR);
+				if ( DetectFrameHeader(recentFrame)) {
+					skip = ftell(m_pFile) - startingOffset;
+					return eRESYNC_FRAME;
+				}
+				fseek(m_pFile, 4, SEEK_CUR);
+			}
+		}
+		else if ( data == -1 ) {
+			fseek(m_pFile, -4, SEEK_CUR);
+			if ( DetectFrameHeader(recentFrame)) {
+				skip = ftell(m_pFile) - startingOffset;
+				return eRESYNC_FRAME;
+			}
+			fseek(m_pFile, 4, SEEK_CUR);
+		}
+		else {
+			// can't do much, next iteration of the loop will read the next integer
+		}
+	}
+
+	return eRESYNC_NONE;
+}
+
+
+bool
+CDaqLowLevelIo::ResyncFrame(
+	int   curFrame,     // last known frame, if -1 then we have no info
+	int&  bytesSkipped)
+{
+	int head[3];
+
+	bytesSkipped = 0;
+	while ( !feof(m_pFile) ) {
+		if ( fread(head, sizeof(int), 3, m_pFile) != 3 ) return false;
+
+		//
+		// Look for a record header
+		//
+		if ( head[0] == -1 ) {
+			bool reasonableFrame = curFrame == -1 || head[1] - curFrame < 20000;
+			bool reasonableCount = head[2] >=0 && head[2] < m_NumEntries;
+
+			if ( reasonableFrame && reasonableCount ) {
+				fseek(m_pFile, -3 * (signed int)sizeof(int), SEEK_CUR);
+				return true;
+			}
+		}
+		//
+		// Look for an end record
+		//
+		else if ( head[0] == -2 ) {
+			return true;            // NEEDS MORE WORK
+		}
+
+		//
+		// Since we've lost allignment, we can't keep reading, we have to
+		// keep looking for a frame header one byte at a time
+		else {
+			fseek(m_pFile, 1 - 3 * sizeof(int), SEEK_CUR);
+			bytesSkipped++;
+		}
+	}
+	return false;
+}
+
+
+void
+CDaqLowLevelIo::Close(void)
+{
+	fclose(m_pFile);
+	m_pFile = 0;
+}
+
+
 bool
 CDaqLowLevelIo::Open(
 	const string &  fname)
 {
+	//
+	// Check if we have already opened the file
+	//
+	if ( m_pFile ) {
+		m_LastError = "file already open";
+		return false;
+	}
 
 	// 
 	// Open the file
 	//
+	m_pFile = fopen(fname.c_str(), "rb");
+	if ( m_pFile == 0 ) {
+		m_LastError = "cannot open file '";
+		m_LastError += fname;
+		m_LastError += "': ";
+		m_LastError += strerror(errno);
+		return false;
+	}
+
+	m_Filename = fname;
+
+	//
+	// Read the header
+	//
+	if ( !ReadHeader(m_pFile) ) {
+		fclose(m_pFile);
+		m_pFile = 0;
+		return false;
+	}
+
+	//
+	// Read channel information 
+	//
+	if ( !ReadChannelInfo(m_pFile) ) {
+		fclose(m_pFile);
+		m_pFile = 0;
+		return false;
+	}
+
+	m_DataOffset = ftell(m_pFile);
+
+	// find the first frame
+	int  maxTries = 2000;
+	int  recHead[3];    // (Code, Frame, Count)
+	
+	while ( maxTries-- ) {
+		if ( fread(recHead, sizeof(int), 3, m_pFile) != 3 ) break;
+
+		if ( recHead[0] == -1 ) {
+			m_FirstFrame = recHead[1];
+			break;
+		}
+	}
+
+	// try to find out the last frame so we have an idea of the total
+	// number of frames in the file
+	fseek(m_pFile, -12, SEEK_END);
+	m_LastFrame = -1;
+	maxTries = 64*1024;
+	while ( maxTries-- ) {
+		if ( fread(recHead, sizeof(int), 3, m_pFile) != 3 ) break;
+
+		if ( recHead[0] == -2 && recHead[2] == DAQ_END_MARK ) { // score !!
+			m_EofStatus = eDAQ_EOF_OK;
+			m_LastFrame = recHead[1];
+			break;
+		}
+		else if ( recHead[0] == -1 && recHead[2] >=0 && recHead[2] < m_NumEntries ) {
+			m_EofStatus = eDAQ_EOF_PARTIAL_MARK;
+			m_LastFrame = recHead[1];
+			break;
+		}
+		else {
+			fseek(m_pFile, -( 3* (signed)sizeof(int) + 1), SEEK_CUR);
+		}
+	}
+
+	if ( m_LastFrame == -1 ) {
+		m_EofStatus = eDAQ_EOF_NOTFOUND;
+	}
+
+	//
+	// No matter what happened with the last frame search, we seek ready to
+	// read data
+	//
+	fseek(m_pFile, m_DataOffset, SEEK_SET);
+
+	//
+	// Now look for the TOC file, if one exists read it
+	//
+	if ( ReadTocFile(fname + ".toc") )
+		m_TocExists = true;
+
+	m_LastError = "";
+	return true;
+}
+
+//////////////////////////////////////////////////////////////////////////////////
+///
+/// This function writes the TOC into a file where it can later be retrieved
+/// without having to go through the DAQ file.
+///
+/// The file contains:
+///		magic (int)
+///		offset of table (int) (i.e., where first frame, offset pair stored
+///		first_frame (int)
+///		last_frame (int)
+///		num_entries in toc (int)
+///		num_framedrops (int)
+///			frame drop 1 len (int)
+///			frame drop 2 len (int)
+///			...
+///     num_droppedframes (int)
+///			dropped frame 1 (int)
+///			dropped frame 2 (int)
+///			...
+///		num_skips (int)
+///			skip 1 len (int)
+///			skip 2 len (int)
+///			...
+///		frame, offset (int, int)
+///		frame, offset (int, int)
+///		...
+///
+bool
+CDaqLowLevelIo::WriteTocFile(
+	const string& fname)
+{
+	FILE *p = fopen(fname.c_str(), "wb");
+	if ( p == 0 ) return false;
+
+	int d1, d2;
+	int tblOfs;
+
+	d1 = DAQ_TOC_MAGIC_NUM;
+	if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
+
+	// again, leave space for table offset
+	if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
+
+	// first frame
+	d1 = this->m_FirstFrame;
+	if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
+
+	// last frame
+	d1 = m_LastFrame;
+	if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
+
+	// num entries
+	d1 = (int)m_Toc.size();
+	if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
+
+	TIntVec::iterator		vi;
+	map<int,int>::iterator	mi;
+
+	// num frame drops
+	d1 = (int)m_FrameDrops.size();
+   	if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
+
+	// frame drops
+	for (vi = m_FrameDrops.begin(); vi != m_FrameDrops.end(); vi++) {
+		d1 = *vi;
+		if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
+	}
+
+	// num dropped frames
+	d1 = (int)m_DroppedFrames.size();
+	if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
+
+	// dropped frames
+	for (vi = m_DroppedFrames.begin(); vi != m_DroppedFrames.end(); vi++) {
+		d1 = *vi;
+		if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
+	}
+
+	// num skips
+	d1 = (int)m_Skips.size();
+	if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
+
+	// skips
+	for (vi = m_Skips.begin(); vi != m_Skips.end(); vi++) {
+		d1 = *vi;
+		if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
+	}
+
+	tblOfs = ftell(p);
+
+	for (mi = m_Toc.begin(); mi != m_Toc.end(); mi++) {
+		d1 = mi->first;
+		d2 = mi->second;
+		if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
+		if ( fwrite(&d2, sizeof(int), 1, p) != 1 ) return false;
+	}
+
+	fseek(p, 4, SEEK_SET);
+	if ( fwrite(&tblOfs, sizeof(int), 1, p) != 1 ) return false;
+
+	fclose(p);
+	return true;
+
+}
+
+
+//////////////////////////////////////////////////////////////////////////////////
+///
+/// This function reads the TOC from a file.  If anything goes wrong with
+/// reading the file, the function quietly returns and doesn't change anything.
+///
+/// The function returns true to indicate it found and successfully loaded a 
+/// toc file, else it returns false
+///
+bool
+CDaqLowLevelIo::ReadTocFile(
+	const string& fname)
+{
+	TIntVec			FrameDrops;
+	TIntVec			DroppedFrames;
+	TIntVec			Skips;
+	int				numEntries;
+	int				i, count, d1, d2;
+
+	//
+	// Cannot read the TOC for a file which we have not yet opened
+	//
+	if ( m_pFile == 0 ) {
+		return false;
+	}
+
 	FILE *p = fopen(fname.c_str(), "rb");
 	if ( p == 0 ) {
 		return false;
 	}
 
+	if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
+	if ( d1 != DAQ_TOC_MAGIC_NUM ) return false;
+
+	// we don't need the offset now, it's for future compatibility
+	if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
+
+	if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
+	if ( m_FirstFrame != d1 ) return false;
+
+	if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
+	if ( m_LastFrame != d1 ) return false;
+
+	if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
+	numEntries = d1;
+
+	if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
+	count = d1;
+	for (i=0; i<count; i++) {
+		if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
+		FrameDrops.push_back(d1);
+	}
+
+	if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
+	count = d1;
+	for (i=0; i<count; i++) {
+		if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
+		DroppedFrames.push_back(d1);
+	}
+
+	if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
+	count = d1;
+	for (i=0; i<count; i++) {
+		if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
+		Skips.push_back(d1);
+	}
+	map<int, int>	Toc;
+
+	vector<int>		frame;
+	vector<int>		offset;
+	for (i=0; i<numEntries; i++) {
+		if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
+		if ( fread(&d2, sizeof(int), 1, p) != 1 ) return false;
+		frame.push_back(d1);
+		offset.push_back(d2);
+	}
+	fclose(p);
+	assert(frame.size() == numEntries);
+	assert(offset.size() == numEntries);
+
+	// all is ok, so copy data
+	m_Toc.clear();
+	for (i=0; i<numEntries; i++) {
+		m_Toc[frame[i]] = offset[i];
+	}
+
+	m_FrameDrops    = FrameDrops;
+	m_DroppedFrames = DroppedFrames;
+	m_Skips         = Skips;
+
+	m_HaveToc = true;
+	return true;
+}
+
+
+//////////////////////////////////////////////////////////////////////////////////
+///
+/// This functions returns information about the file's integrity. It can 
+/// only be called after CheckIntegrity has successfully returned.
+/// 
+/// The function returns (in the provided arguments) two arrays of 
+/// integers. The first array has an element for each time time frame drops 
+/// were encountered. The actual array element is the number of frames that 
+/// were dropped. Similarly, the second array has the number of times data 
+/// has to be skipped to re-sync; one element is used for each successful 
+/// re-sync attempt, the actual element value is the number of bytes that 
+/// had to be skipped. 
+///
+bool 
+CDaqLowLevelIo::QueryIntegrityValues(
+	vector<int>& drops, 
+	vector<int>& dropedFrm,
+	vector<int>& skips,
+	TDaqEofStatus& eofStat)
+{
+	if ( m_HaveToc == false ) return false;
+
+	drops     = m_FrameDrops;
+	dropedFrm = m_DroppedFrames;
+	skips     = m_Skips;
+	eofStat   = m_EofStatus;
+
+	return true;
+}
+
+
+//////////////////////////////////////////////////////////////////////////////////
+///
+/// This function verifies the integrity of the DAQ file by going through it 
+/// and checking all records for consistency. In addition, the function builds 
+/// a table of contents that allows better random access to the contents 
+/// of the file. Once this function has returned successfully, the functions 
+/// that allow access to the subset of the frames can be used.
+///
+/// By default, the table of contents is kept in memory, however, if the
+/// buildTocFile argument is true, then the function will write the TOC into
+/// a separate file that has the same name as the DAQ file but with the
+/// extension .toc added after the name.  The TOC file is a binary
+/// representation of the toc and integrity values.  Note that no check
+/// is made for prior existence of the TOC file.  
+///
+/// By default, any errors that occur while tring to create the TOC file
+/// are silently ignored.  If the forceFileCreate flag is set, then any
+/// such error is treated as a fatal error and the function will return
+/// an error code.
+///
+/// The function returns true to indicate the file is ok (or has problems 
+/// that can be fixed) or false to indicate an unrecoverable error. 
+/// To determine the existence of dropped frames and any other recoverable 
+/// error, use the QueryIntegrityValues() function.
+///
+/// The function will call, within its context, the progress callback as 
+/// well as the error callback. Note that the error callback is only called 
+/// for recoverable errors which allow  the function to continue working.
+/// Unrecoverable errors that cause the function to return false do not 
+/// trigger the error callback.
+///
+bool
+CDaqLowLevelIo::CheckIntegrity(
+	bool	writeTocFile,
+	bool	forceFileCreate
+)
+{
+	char	buf[256];
+	int		oldFrame   = -1;
+	int		numFrames  = 0;
+	map<int, TFrameInfo>  tempToc;
+	bool    done = false;
+
+	//
+	// Go through the file and build the TOC.  Only frames with data that
+	// have shown no data skips or other problems are kept
+	// 
+	m_FirstFrame = -1;
+	while ( !done ) {
+		int  curFrame = -1;     // frame for current record
+		int  numChan;           // number of channels in current record
+		int  skip = 0;
+		int  frameOffset;
+
+		CDaqLowLevelIo::TReadErrorCode  rcode;
+
+		rcode = ReadFrameHeader(-1, curFrame, numChan, skip);
+		if ( rcode == eDAQ_READ_OK || rcode == eDAQ_READ_EOF ) {
+			if ( m_FirstFrame == -1 ) m_FirstFrame = curFrame;
+
+			if ( (numFrames % 720) == 0 ) (*m_UserProgrFunc)(m_Filename.c_str(), 
+						numFrames, m_LastFrame - m_FirstFrame);
+
+			frameOffset = ftell(m_pFile) - 12;
+
+			numFrames++;
+			if ( rcode == eDAQ_READ_EOF ) {
+				int tot = m_LastFrame - m_FirstFrame;
+				(*m_UserProgrFunc)(m_Filename.c_str(), tot, tot);
+				break;
+			}
+		}
+		else if ( rcode == eDAQ_READ_ERROR ) {	// bad code read, couldn't resync
+			sprintf(buf, "Could not recover after error at file offset %d", ftell(m_pFile));
+			m_LastError = buf;
+			return false;
+		}
+		else if ( rcode == eDAQ_READ_FAILED ) {	// fread  failed
+			sprintf(buf, "Could not read data near offset %d", ftell(m_pFile));
+			m_LastError = buf;
+			return false;
+		}
+		else {
+			m_LastError =  "General read error";
+			return false;
+		}
+
+		// print warning and keep track of frame skips
+		if ( skip > 0 ) {
+			m_Skips.push_back(skip);
+			sprintf(buf, "Data error before %d, resync'ed successfuly", ftell(m_pFile));
+			(*m_UserErrorFunc)(m_Filename.c_str(), buf);
+		}
+
+		// Now read information about data in this record and skip over data portion
+		int ch;
+		bool problems = false;
+
+		ch = 0;
+		while ( ch < numChan ) {
+			int chId;
+			int ofs=ftell(m_pFile);		// to keep track of where errors may occur
+
+			if ( fread(&chId, sizeof(int), 1, m_pFile) != 1 ) {
+				if ( m_EofStatus == eDAQ_EOF_NOTFOUND ) { // OK, we had no end of file
+					m_LastFrame = curFrame;
+					done        = true;
+					(*m_UserErrorFunc)(m_Filename.c_str(), "reached EOF with no end-marker");
+					break;
+				}
+				else if ( m_EofStatus == eDAQ_EOF_PARTIAL_MARK && curFrame == m_LastFrame ) {
+					m_LastFrame = curFrame;
+					done        = true;
+					break;
+				}
+				else {
+					sprintf(buf, "Couldn't read file, premature end");
+					m_LastError = buf;
+					return false;
+				}
+			}
+			if ( chId >= m_NumEntries || chId < 0 ) {
+				// NearDump(m_pFile, ofs);
+				TResyncOutcome rcode;
+				int errOfs = ftell(m_pFile);
+
+				problems = true;
+				rcode = ResyncChannel(curFrame, skip);
+				if ( rcode == eRESYNC_CHANNEL || rcode == eRESYNC_FRAME ) {
+					sprintf(buf, "Invalid channel %d @ ofs %d, resync'ed ok", chId, errOfs);
+					(*m_UserErrorFunc)(m_Filename.c_str(), buf);
+					break;
+				}
+				else {
+					sprintf(buf, "Invalid channel %d @ ofs %d, could not resync", chId, errOfs);
+					m_LastError = buf;
+					return false;
+				}
+			}
+			else {
+				CDaqChannelInfo& ch = m_Channels[chId];
+				int  skip = ch.GetRecSize();
+				fseek(m_pFile, skip, SEEK_CUR);
+			}
+			ch++;
+
+			// This code addresses a specific bug in the DAQ that would write
+			// the wrong number of channels in the frame header.  What happens in
+			// such cases is taht the next integer, instead of -1 (as the frame 
+			// header) would be a channel number containing valid data.  
+			if ( ch == numChan ) {
+				int  oldChId = chId;
+				int  numRead = (int)fread(&chId, sizeof(int), 1, m_pFile);
+				bool okChan  = chId > oldChId && chId < m_NumEntries;
+
+				if ( numRead == 1 && okChan ) {
+					sprintf(buf, "Found extra channel %d in frame %d", chId, curFrame);
+					(*m_UserErrorFunc)(m_Filename.c_str(), buf);
+					ch--;
+					fseek(m_pFile, -4, SEEK_CUR);
+					continue;
+				}
+				else {
+					fseek(m_pFile, -4, SEEK_CUR);
+				}
+			}
+		}
+
+		if ( !problems ) { // we only want frames for which data was intact
+			TFrameInfo info;
+
+			info.numChan      = numChan;
+			info.ofs          = frameOffset;
+			tempToc[curFrame] = info;
+		}
+	}
+
 	// 
-	// Read the header and print, if needed
+	// At this point, we have the temporary TOC file which we can use
+	// to calculate actual frame drops.  AFter this is done, we can copy
+	// into the final TOC only frames that carry data
 	//
-	if ( !ReadHeader(p) ) {
+	{
+		int frm = m_FirstFrame;
+		int old = -1;
+		int curFrame;
+
+		while ( frm < m_LastFrame ) {
+			if ( tempToc.find(frm) != tempToc.end() ) {// if it's in the map, it was in the file 
+				curFrame = frm;
+				if ( old != -1 && curFrame - old > 1 ) {
+					int len = (curFrame-old) - 1;
+					m_FrameDrops.push_back(len);
+				}
+				old = curFrame;
+			}
+			else {
+				m_DroppedFrames.push_back(frm);
+			}
+			frm++;
+		}
+	}
+
+	// Now copy into the final TOC entries with data
+	map<int, TFrameInfo>::iterator pTempToc;
+
+	m_Toc.clear();
+	for (pTempToc = tempToc.begin(); pTempToc != tempToc.end(); pTempToc++) {
+		if ( pTempToc->second.numChan > 0 ) m_Toc[pTempToc->first] = pTempToc->second.ofs;
+	}
+
+	m_HaveToc = true;
+
+	if ( writeTocFile ) {
+		string nm = m_Filename + ".toc";
+		if ( WriteTocFile(nm) == false && forceFileCreate ) {
+			sprintf(buf, "Could not build toc on file %s: %s", nm.c_str(), strerror(errno));
+			m_LastError = buf;
 			return false;
 		}
+	}
 
-	printf("Title     = %s\n", m_Title.c_str());
-	printf("Date      = %s", m_Date.c_str());
-	printf("Subj      = %s\n", m_Subj.c_str());
-	printf("Run       = %s\n", m_Run.c_str());
-	printf("RunInst   = %s\n", m_RunInst.c_str());
-	printf("Channels  = %d\n", m_NumEntries);
-	printf("Frequency = %d\n", m_Frequency);
 
+	m_LastError = "";
+	return true;
+}
+
+
+///////////////////////////////////////////////////////////////////////////////
+///
+/// This function will read data for a set of channels from the DAQ file
+/// into memory buffers.  The function does not require a table of contents
+/// to operate.
+///
+/// The input vectors specify which channels to read.  The first vector
+/// simply lists the channel identifiers to be read, and the second 
+/// vector contains the buffers to receive the corresponding data.
+/// Both vectors should be the same size, unless the pointer to the 
+/// first vector is 0 which implies the whole file should be read.  In that
+/// case, the second vector should be sized large enough to receive
+/// all channels.
+///
+/// The function returns true to indicate all data was read, or false to
+/// indicate some problem occurred.
+///
+bool 
+CDaqLowLevelIo::LowLevelDataRead(
+	const vector<int>*  pWhichChannels,  
+	vector<CDaqBuffer>& userStorage
+)
+{
 	//
-	// Read channel information and print, if needed
+	// Standard error checking
 	//
-	if ( !ReadChannelInfo(p) ) {
+	if ( m_pFile == 0 ) {
+		m_LastError = "uninitialized class instance";
 		return false;
 	}
-	int ch;
 
-	printf("Channels:\n");
-	printf("  Id         Name                           Items Type  Rate  VarLen\n");
-	for (ch=0; ch<m_NumEntries; ch++) {
-		printf("%4d  %-36s %5d    %c     %d    %d\n",
-			m_Channels[ch].m_Id, m_Channels[ch].m_Name.c_str(),
-			m_Channels[ch].m_Items, m_Channels[ch].m_Type,
-			m_Channels[ch].m_CapRate, m_Channels[ch].m_VarLen ? 1 : 0);
+	if ( pWhichChannels && pWhichChannels->size() != userStorage.size() ) {
+		m_LastError = "inconsistently sized input arguments";
+		return false;
 	}
 
-	m_DataOffset = ftell(p);
+	char buf[256];              // buffer for writing messages
+
+	// temporary storage; use a vector so we don't have to worry about
+	// memory leaks
+	vector<unsigned char>  tempSpace;
+	tempSpace.resize(cMaxDaqRec);
+	void *pTempSpace = (void *)&tempSpace[0];
 
 	//
-	// Go through the file and build the TOC
+	// Create a faster data structure to tell us which channels are needed.
+	// Once done, needChan[channelId] is -1 if we don't need, or the index
+	// within the user specified 'data' vector of where data should be stored
 	//
-	while ( 1 ) {
-		int  three[3];	// (Code, Frame, Count)
+	vector<int>  needChan;
+	if ( pWhichChannels ) {
+		unsigned     i;
 
-		if ( fread(three, sizeof(int), 3, p) != 3 ) {
-			fprintf(stderr, "Couldn't read file, premature end or messed up file\n");
+		needChan.resize(m_NumEntries, -1);
+		for (i=0; i<pWhichChannels->size(); i++) {
+			if ( (*pWhichChannels)[i] < 0 || (*pWhichChannels)[i] >= m_NumEntries) {
 				return false;
 			}
+			needChan[(*pWhichChannels)[i]] = i;
+		}
+	}
+	else {
+		int i;
+		for (i=0; i<m_NumEntries; i++)
+			needChan.push_back(i);
+	}
+
+	//
+	// Go through the file collecting the data we need.
+	//
+	int numFrames = 0;
+	fseek(m_pFile, m_DataOffset, SEEK_SET);
+	while ( 1 ) {
+		int  recHead[3];        // (Code, Frame, Count)
+		int  curFrame = -1;     // frame for current record
+		int  numChan;           // number of channels in current record
 
-		if ( three[0] < 0 ) {	// some marker
-			if ( three[0] == -1 ) {		// frame
-				m_Toc[three[1]] = ftell(p);	// add to TOC
-				printf("F=%d ", three[1]);
+		if ( fread(recHead, sizeof(int), 3, m_pFile) != 3 ) {
+			sprintf(buf, "Couldn't read file, premature end or messed up file");
+			m_LastError = buf;
+			return false;
 		}
-			else if ( three[0] == -2 ) {	// end marker
-				if ( three[2] == DAQ_END_MARK ) {
-					break;		/// <-----------  END 
+
+		if ( recHead[0] == -1 ) {           // This is a frame marker
+			curFrame  = recHead[1];
+			numChan   = recHead[2];
+			numFrames++;
+
+			// print info every 30 seconds of collection (30*240=720)
+			if ( (numFrames % 720) == 0 ) 
+				(*m_UserProgrFunc)(m_Filename.c_str(), numFrames, m_LastFrame - m_FirstFrame);
+
 			}
+			else if ( recHead[0] == -2 ) {      // This is an end marker
+				curFrame = recHead[1];
+				if ( recHead[2] == DAQ_END_MARK ) {
+					break; 
 				}
 				else {
-				fprintf(stderr, "Unexpected code %d at line %d\n", three[0], __LINE__);
+					sprintf(buf, "Unexpected secondary code %d at line %d", recHead[0], 
+							__LINE__);
+					m_LastError = buf;
 					return false;
 				}
 			}
 			else {
-			int chId = three[0];
+				sprintf(buf, "Unexpected header code %d at file offset %d", 
+						recHead[0], ftell(m_pFile));
+				(*m_UserErrorFunc)(m_Filename.c_str(), buf);
+
+				int skip;
+				int ofs = ftell(m_pFile);
+				if ( ResyncFrame(curFrame, skip) == false ) {
+					sprintf(buf, "Could not recover after error at file offset %d", ofs);
+					m_LastError = buf;
+					return false;
+				}
+				else {
+					continue;
+				}
+			}
+
+			// Now read the data for this record
+			int idx;
+			for (idx=0; idx<numChan; idx++) {
+				int chId;
+
+				if ( fread(&chId, sizeof(int), 1, m_pFile) != 1 ) {
+					sprintf(buf, "Couldn't read file, premature end");
+					m_LastError = buf;
+					return false;
+				}
 				if ( chId >= m_NumEntries ) {
-				fprintf(stderr, "Invalid channel %d\n", chId);
-				NearDump(p);
+					sprintf(buf, "Encountered invalid channel %d in data area", chId);
+					m_LastError = buf;
 					return false;
 				}
 
 				CDaqChannelInfo& ch   = m_Channels[chId];
-			int  skip = DataSize(ch);
-			printf("Channel %d found, %d bytes\n", three[0], skip);
+				int              size = ch.GetRecSize();
+				if ( needChan[chId] != -1 ) {	// it is -1 when we don't need it
+					CDaqBuffer&  daqBuf = userStorage[needChan[chId]];
+
+					if ( size > cMaxDaqRec ) {
+						sprintf(buf, "Daq record size too big, increase \"cMaxDaqRec\"");
+						m_LastError = buf;
+						return false;
+					}
+
+					size_t n = fread(pTempSpace, size, 1, m_pFile);
+					if ( n != 1 ) {
+						sprintf(buf, "read data for channel %d near frame %d failed",
+						chId, curFrame);
+						m_LastError = buf;
+						return false;
+					}
+					daqBuf.Append(pTempSpace, size, ch.GetItemCount());
+				}
+				else {
+				// skip the data
+					fseek(m_pFile, size, SEEK_CUR);
+				}
+			}
+		}
+
+	return true;
+}
+
+
+///////////////////////////////////////////////////////////////////////////////////////////
+///
+/// This function is similar to LowLevelDataRead but instead of reading the data 
+/// into memory buffers it copies the data into a set of output files for which the 
+/// user has already obtained FILE handles.
+///
+/// The function will scan the whole file and store the samples of each channel 
+/// specified in chans into the corresponding file. The options argument controls 
+/// what the function does when it encounters dropped frames or differentially 
+/// stored channels. By default, dropped frames are ignored and only samples in 
+/// the DAQ file are written to the output, even for differential data.
+///
+/// If the eEXPAND_DIFFERENTIAL option is set, then when encountering a differential 
+/// channel, the function will write out the data at the maximum sampling frequency 
+/// by using the most recent value for samples that were intentionally skipped.
+///
+/// If the eFILL_MISSING option is set, then the function will detect dropped 
+/// frames and write out replacement values to ensure that all the output file(s) 
+/// contain the appropriate number of samples. Data for dropped frames is generated 
+/// by using the most recent data of the same channel (eFILL_MISSING_ORDER_0), linear 
+/// interpolation (eFILL_MISSING_ORDER_1) or cubic interpolation (eFILL_MISSING_ORDER_2).
+///
+/// The output files can be written in ASCII or binary.  The argument FileHandles
+/// is a vector of FILE pointers that should already have been opened successfully.
+/// The file handles must match the binary/ascii option specified in options. 
+/// When asking for data in ASCII, 
+/// each sample will produce a line in the output file. That line contains the 
+/// items for the sample separated by a space. When asking for data in binary, 
+/// the data is written in raw binary format. For example, a channel with 3 
+/// floats per sample will occupy 12 bytes per sample.
+///
+/// The function will not close the file handles before returning, that is the
+/// responsibility of the caller.
+///
+/// The final option controlling the data written to the file is the 
+/// eINCLUDE_FRAME option. If this option is specified, a frame number is 
+/// written before each sample in the output file. In ASCII mode, the frame is 
+/// written as an integer at the beginning of each line and in binary mode a 
+/// four byte integer is written before each sample.
+///
+/// The function returns true to indicate that it successfully went through 
+/// the file and wrote the data, or false if it encountered an unrecoverable error. 
+///
+bool 
+CDaqLowLevelIo::LowLevelDataCopy(
+	const TIntVec*  pWhichChannels,	///< List of channels to write to the output
+	TFileVec&       FileHandles,	///< Open file handles to use for corresponding channels
+	TConvertOptions options			///< Options controlling how data is written
+)
+{
+	//
+	// Standard error checking
+	//
+	if ( m_pFile == 0 ) {
+		m_LastError = "uninitialized class instance";
+		return false;
+	}
+
+	if ( m_HaveToc == false ) {
+		m_LastError = "need to buld table of contents first";
+		return false;
+	}
+
+	if ( pWhichChannels && pWhichChannels->size() != FileHandles.size() ) {
+		m_LastError = "inconsistently sized input arguments";
+		return false;
+	}
+
+	char buf[256];              // buffer for writing messages
+
+	// temporary storage; use a vector so we don't have to worry about
+	// memory leaks
+	vector<unsigned char>  tempSpace;
+	tempSpace.resize(cMaxDaqRec);
+	void *pTempSpace = (void *)&tempSpace[0];
+	vector<CDaqBuffer>   latestData;		// latest sample of data
+
+	//
+	// Create a faster data structure to tell us which channels are needed.
+	// Once done, needChan[channelId] is -1 if we don't need, or the index
+	// within the user specified 'data' vector of where data should be stored
+	//
+	vector<int>  needChan;
 
-			fseek(p, skip, SEEK_CUR);
+	if ( pWhichChannels ) {
+		unsigned     i;
+		CDaqBuffer   temBuf;
 
+		needChan.resize(m_NumEntries, -1);
+		latestData.resize(m_NumEntries, temBuf);
+
+		for (i=0; i<pWhichChannels->size(); i++) {
+			if ( (*pWhichChannels)[i] < 0 || (*pWhichChannels)[i] >= m_NumEntries) {
+				return false;
+			}
+			needChan[(*pWhichChannels)[i]] = i;
+		}
+	}
+	else {
+		int i;
+		CDaqBuffer   temBuf;
+
+		latestData.resize(m_NumEntries, temBuf);
+		for (i=0; i<m_NumEntries; i++)
+			needChan.push_back(i);
+	}
+
+	int                  frm = m_FirstFrame;
+
+	while ( frm <= m_LastFrame ) {
+		map<int, int>::const_iterator  pTocEn;
+		vector<int>::const_iterator    pDrpFrm;
+		bool                           curFrameLost = false;
+		int	                           numChan;
+
+		// determine if this is a frame that was dropped
+		for (pDrpFrm = m_DroppedFrames.begin(); pDrpFrm != m_DroppedFrames.end(); pDrpFrm++){
+			if ( frm == *pDrpFrm ) {
+				curFrameLost = true;
+				break;
+			}
+		}
+
+		if ( curFrameLost == false ) {
+			pTocEn = m_Toc.find(frm);	// does it have data ?
+			if ( pTocEn == m_Toc.end() ) {	// no data for this frame
+				// do nothing
+			}
+			else {
+				int actualFrameRead;
+				int	dummy;
+				TReadErrorCode rcode;
+
+				rcode = ReadFrameHeader(frm, actualFrameRead, numChan, dummy);
+				if ( rcode != eDAQ_READ_OK ) {
+					sprintf(buf, "ReadFrameHeader returned %d",  rcode);
+					m_LastError = buf;
+					return false;
+				}
+				assert(frm == actualFrameRead);
+
+				ReadDataForOneFrame(numChan, needChan, latestData, pTempSpace);
+				for (int ch=0; ch < m_NumEntries; ch++) {
+					CDaqChannelInfo& chInf = m_Channels[ch];
+					int              size  = chInf.GetRecSize();
+					size_t           out;
+					
+					int nn = needChan[ch];
+					out = fwrite(latestData[needChan[ch]].GetDataPtr(), size, 1, 
+								FileHandles[needChan[ch]]);
+				}
+			}
+		}
+		else {	// current frame lost
+			// Produce data for current frame based on prior values
+			for (int ch=0; ch < m_NumEntries; ch++) {
+				CDaqChannelInfo& chInf = m_Channels[ch];
+				int              size  = chInf.GetRecSize();
+				size_t           out;
+
+				out = fwrite(latestData[needChan[ch]].GetDataPtr(), size, 1, 
+					FileHandles[needChan[ch]]);
+			}
 		}
+		frm++;
 	}
 
 	return true;
+}
+
+
+///////////////////////////////////////////////////////////////////////////////////////////
+///
+/// This function is similar to LowLevelDataRead but provides additional options for 
+/// data management and selection of a range of frames to read. Also, unlike 
+/// LowLevelDataRead, this function can only be called if the class has access
+/// to the table of contents (either built with the CheckDataIntegrity function or
+/// read from a prior run). Finally, the function always expands differentially 
+/// stored channels.
+///
+/// The function reads the specified list of channels into memory buffers, 
+/// fills in data for dropped frames, can optionally re-sample the data, and 
+/// if specified, read a subset of the data based on a start and end frame.
+///
+/// The chans vector lists the identifiers for which channels to read. The 
+/// data vector contains data buffers onto which to store the samples of the 
+/// corresponding channels. 
+///
+/// The subsample vector indicates if the corresponding 
+/// channel should be sub-sampled. The number should be an integer greater than
+/// or equal to 0.  It is interpreted as follows:
+/// If 0, then a data sample is written as often as it was collected in the file
+/// If greater than zero, the function will output data every so many fbs
+/// frames, starting with the first frame on which the channel was collected,
+/// independent of how often data was sampled.  The corresponding entry in the
+/// options array specifies how to interpolate or extrapolate data.
+///
+/// Consider the following example.  The first line indicates during which
+/// fbs frames a channel was collected.  The second line shows the actual
+/// fbs frame numbers.  The remaining lines have an X at each frame at which
+/// data would be output depending on the value of the respective subsample
+/// element.  Note that each time during which the function outputs a value
+/// but there was no corresponding collection at runtime will result in
+/// the application of interpolation according to the options array.
+///
+/// Sample collected :       X           X           X           X
+/// FBS frm          : 0  1  2  3  4  5  6  7  8  9  10 11 12 13 14 15
+/// subsample = 0    :       X           X           X           X
+/// subsample = 1    :       X  X  X  X  X  X  X  X  X  X  X  X  X  X
+/// subsample = 2    :       X     X     X     X     X     X     X
+/// subsample = 3    :       X        X        X        X        X
+/// subsample = 4    :       X           X           X           X
+///
+/// Finally, note that for differentially stored data, the extrapolation 
+/// method defaults to zero-hold, no matter what specification is given.
+///
+/// The options argument is used to specify how to deal with dropped frames 
+/// and as such, the function is only concerned with the eFILL_MISSING_XXXXXX 
+/// options. In particular, specifying eFILL_MISSING will fill in dropped 
+/// frames by using a zero-hold technique by default. Specify any 
+/// of eFILL_MISSING_ORDER_? constants to modify the default behavior. 
+///
+/// The frm1 and frm2 arguments specify the first/last frame to read, inclusively.
+/// Specify -1 for either, to read from the start or to the end of file respectively.
+/// The frame numbers are 0 based, so the first frame would be 0.
+///
+bool 
+CDaqLowLevelIo::GetFullData(
+	const TIntVec*  pWhichChannels,      
+	TDataVec&       data,       
+	const TIntVec&  subsample,  
+	TConvertOptions options,    
+	int             frm1,       
+	int             frm2)       
+{
+	//
+	// Standard error checking
+	//
+	if ( m_pFile == 0 ) {
+		m_LastError = "uninitialized class instance";
+		return false;
+	}
+
+	if ( frm1 > GetFrames() || frm2 > GetFrames() ) {
+		m_LastError = "invalid frame range";
+		return false;
+	}
+
+	// 
+	// Adjust frame numbers so they reflect fbs frames
+	//
+	if ( frm1 < 0 ) frm1 = GetFirstFrame(); else frm1 += GetFirstFrame();
+	if ( frm2 < 0 ) frm2 = GetLastFrame(); else frm2 += GetLastFrame();
+
+	char buf[256];              // buffer for writing messages
+
+	//
+	// Create a faster data structure to tell us which channels are needed.
+	// Once done, needChan[channelId] is -1 if we don't need, or the index
+	// within the user specified 'data' vector of where data should be stored
+	//
+	vector<int>  needChan;
+	if ( pWhichChannels ) {
+		unsigned     i;
+
+		needChan.resize(m_NumEntries, -1);
+		for (i=0; i<pWhichChannels->size(); i++) {
+			if ( (*pWhichChannels)[i] < 0 || (*pWhichChannels)[i] >= m_NumEntries) {
+				return false;
+			}
+			needChan[(*pWhichChannels)[i]] = i;
+		}
+	}
+	else {
+		int i;
+		for (i=0; i<m_NumEntries; i++)
+			needChan.push_back(i);
+	}
+	// temporary storage; use a vector so we don't have to worry about
+	// memory leaks
+	vector<unsigned char>  tempSpace;
+	tempSpace.resize(cMaxDaqRec);
+	void *pTempSpace = (void *)&tempSpace[0];
+
+	//
+	// Various other variables
+	//
+	vector<CDaqBuffer>   latestData;		// latest sample of data
+	vector<int>          notNeededNow;		// helps maintain when a sample is needed
+	vector<bool>         latestFromThisFrm;	// when true, latestData defined on this frame
+//	int                  startOfs;
+	map<int, int>::const_iterator	mapIter;
+	int                  curFrame;
+	int                  numChan;           // number of channels in current record
+	int                  bytesSkipped;
+
+	CDaqLowLevelIo::TReadErrorCode rcode;
+
+	rcode = ReadFrameHeader(frm1, curFrame, numChan, bytesSkipped);
+	if ( rcode != eDAQ_READ_OK ) {
+		sprintf(buf, "ReadFrameHeader returned %d",  rcode);
+		m_LastError = buf;
+		return false;
+	}
+
+	// -> Read data into latestData, make sure we have read one of everything
+
+	while ( curFrame <= frm2 ) {
+		size_t chan;
+
+		for (chan = 0; chan < m_Channels.size(); chan++) {
+			ReadDataForOneFrame(numChan, needChan, latestData, pTempSpace);
+
+		}
+	}
+
+	return false;
+}
+
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// This function reads channel data for the current record.  It assumes that
+// the file pointer points to the first record immediately after the
+// frame header.  It will read as many samples as indicated in the 'numEntries'
+// input argument.  The needChannel array indicates if a given channel id
+// is needed or not.  It is dimensioned according to the total number of
+// channels available in the DAQ file.
+// The storage array holds the data for this frame, one entry per channel. 
+// Note that indexing the needChannel array is different than the storage
+// array, in particular, needChannel should be m_NumEntries long but storage
+// is dimensioned long enough to keep track of the needed entires as specified
+// in needChannel.  
+//
+// As an example, assume that the number of channels in the file is 20 (0..19).
+// The value of m_NumEntries should be 20 also.  The array needChannel should
+// be 20 ints long.  It will look something like this:
+// -1 -1 -1 0 -1 1 -1 -1 2 3 4 -1 -1 -1 -1 5 6 7 -1 -1
+// indicating that we care about channels 3, 5, 8, 9, 10, 15, 16, and 17.
+// The storage array should be 8 in length, with channel 3 in position 0,
+// channel 5 in position 1, channel 8 in position 2, channel 9 in position 3 etc.
+// In effect, if needChannel[i] is >=0, its data should be stored
+// in storage[needChannel[i]]
+// 
+// Finally, pTempSpace is a buffer where the function can temporarily read
+// values.  We pass it down to avoid repeated memory allocations.
+//
+CDaqLowLevelIo::TReadErrorCode
+CDaqLowLevelIo::ReadDataForOneFrame(
+	int           numEntriesInThisRecord,
+	vector<int>   needChannel,
+	TDataVec&     storage,
+	void*         pTempSpace
+)
+{
+	int idx;
+	char buf[256];
+
+	for (idx=0; idx<numEntriesInThisRecord; idx++) {
+		int chId;
+		if ( fread(&chId, sizeof(int), 1, m_pFile) != 1 ) return eDAQ_READ_ERROR;
+
+		if ( chId >= m_NumEntries ) {
+			sprintf(buf, "Encountered invalid channel %d in data area", chId);
+			m_LastError = buf;
+			return eDAQ_READ_BAD_DATA;
+		}
+
+		CDaqChannelInfo& ch   = m_Channels[chId];
+		int              size = ch.GetRecSize();
+		if ( needChannel[chId] != -1 ) {	// it is -1 when we don't need it
+			CDaqBuffer&  daqBuf = storage[needChannel[chId]];
+
+			daqBuf.AllocSpace(0);		// clear contents
+			if ( size > cMaxDaqRec ) {
+				sprintf(buf, "Daq record size too big, increase \"cMaxDaqRec\"");
+				m_LastError = buf;
+				return eDAQ_READ_INTERNAL_ERROR;
+			}
+
+			size_t n = fread(pTempSpace, size, 1, m_pFile);
+			if ( n != 1 ) {
+				sprintf(buf, "read data for channel %d failed", chId);
+				m_LastError = buf;
+				return eDAQ_READ_ERROR;
+			}
+			daqBuf.Append(pTempSpace, size, ch.GetItemCount());
+		}
+		else {
+			// skip the data
+			fseek(m_pFile, size, SEEK_CUR);
+		}
+	}
+	return eDAQ_READ_OK;
+}
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// This function reads the header for the upcoming frame.  It will verify
+// it's a correct header and try to re-sync reading if the next bytes do
+// not make up a reasonable header.
+//
+// The frameToRead argument indicates which frame we are trying to
+// read.  If not set to -1, the function will assume a table of contents
+// exists and will seek to the specific offset within the file, but if
+// frameToRead is set to -1, the function simply assumes the file 
+// pointer is set to read the header next.
+//
+// Upon a successful return, curFrame is the fbs frame number that
+// was read and numChan is the number of channels stored for that frame.
+//
+// The skip variable indicates how many bytes had to be skipped in
+// order to find a good header.  U
+// Return: OK, EOF, SKIPPED, ERROR
+CDaqLowLevelIo::TReadErrorCode
+CDaqLowLevelIo::ReadFrameHeader(
+	int     frameToRead,// which frame we anticipate reading, use -1 to read next available 
+	int&    curFrame,	// which frame was read
+	int&    numChan,	// number of channels in this frame
+	int&    skip		// how many bytes were skipped (if >0 indicates some corruption)
+)
+{
+	int    recHead[3];        // (Code, Frame, Count)
+	char   buf[256];          // buffer for writing messages
+	
+	skip = 0;	// by default, we expect no skipping
+
+	// We run a loop because we may have to try again in case of frame
+	// drops or other file corruption
+	for ( ; ; ) {
+		if ( frameToRead >= 0  ) {
+			int startOfs;
+			map<int, int>::const_iterator	mapIter;
+
+			mapIter = m_Toc.find(frameToRead);
+			if ( mapIter == m_Toc.end() ) {
+				return eDAQ_NO_SUCH_FRAME;
+			}
+			startOfs = mapIter->second;
+			fseek(m_pFile, startOfs, SEEK_SET);
+		}
+
+		// Read the frame header; if this fails, we are really in trouble
+		if ( fread(recHead, sizeof(int), 3, m_pFile) != 3 ) {
+			sprintf(buf, "Couldn't read file, premature end or messed up file");
+			m_LastError = buf;
+			return eDAQ_READ_ERROR;
+		}
+
+		// Most of the time, this is what we'll get
+		if ( recHead[0] == -1 ) {  // Frame marker 
+			curFrame  = recHead[1];
+			numChan   = recHead[2];
+
+			// print info every 30 seconds of collection (30*240=720)
+			if ( (curFrame % 720) == 0 ) {
+				(*m_UserProgrFunc)(m_Filename.c_str(), curFrame - m_FirstFrame, 
+					m_LastFrame - m_FirstFrame);
+			}
+			return eDAQ_READ_OK;
+		}
+
+		if ( recHead[0] == -2 ) {      // This is an end marker, so we're done
+			curFrame = recHead[1];
+			if ( recHead[2] != DAQ_END_MARK ) {
+				// problem
+			}
+			return eDAQ_READ_EOF;
+		}
+
+		// If we get here, then what we expected to be a frame header is not.  
+		// Most likely some data didn't make it and we are out of sync with
+		// where things are.  We will try to resynchronize by looking for
+		// a reasonable frame marker at some point ahead 
+		sprintf(buf, "Expected header code (-1) but found %d at fileofs %d", 
+					recHead[0], ftell(m_pFile));
+		(*m_UserErrorFunc)(m_Filename.c_str(), buf);
+
+		int ofs = ftell(m_pFile);
+		if ( ResyncFrame(curFrame, skip) == false ) {
+			sprintf(buf, "Could not recover after error at file offset %d", ofs);
+			m_LastError = buf;
+			return eDAQ_READ_FAILED;
+		}
+		frameToRead = -1;		// no seeking needed after a resync
+	}
 
 }
\ No newline at end of file

DaqIOLib.vcproj

@@ -28,7 +28,10 @@
 				Detect64BitPortabilityProblems="TRUE"
 				DebugInformationFormat="4"/>
 			<Tool
-				Name="VCCustomBuildTool"/>
+				Name="VCCustomBuildTool"
+				CommandLine="copy DaqIo.h ..\include"
+				AdditionalDependencies="DaqIo.h"
+				Outputs="..\include\DaqIo.h"/>
 			<Tool
 				Name="VCLibrarianTool"
 				OutputFile="$(OutDir)/DaqIOLib.lib"/>

DaqIo.h

@@ -5,17 +5,91 @@
 
 using namespace std;
 
+class CDaqBuffer;
+class CDaqChannelInfo;
+
 
 #define DAQ_MAGIC_NUM_VER_2_0 0x2C3D4E5F
 #define DAQ_MAGIC_NUM_VER_2_1 0x2C3D4E6F
 #define DAQ_MAGIC_NUM_VER_2_2 0x2C3D4E7F
 #define DAQ_END_MARK          0x1A2B2C3D
+#define DAQ_TOC_MAGIC_NUM     0x2C3ABCDE
+
+typedef unsigned char       TDaqByte;
+typedef vector<int>         TIntVec;
+typedef vector<string>      TStrVec;
+typedef vector<CDaqBuffer>  TDataVec;
+typedef vector<FILE *>      TFileVec;
+typedef vector<CDaqChannelInfo> TChanInfoVec;
+
+
+const int cMaxDaqRec   = 96*1024;   ///< maximum size of data per DAQ record
+const int cMaxChannels = 500;       ///< maximum number of channels we can handle
 
+///////////////////////////////////////////////////////////////////////////////////
+///
+/// This enumeration encapsulates various conversion options
+/// Appropriate operators have been defined so combining options
+/// can be done by using + or |
+///
+typedef enum {
+	eNO_OPTION            = 0,
+	eEXPAND_DIFFERENTIAL  = 1,      
+	eFILL_MISSING         = 2,      
+	eFILL_MISSING_ORDER_0 = 4,      
+	eFILL_MISSING_ORDER_1 = 8,      
+	eFILL_MISSING_ORDER_2 = 16,     
+	eBINARY_FORMAT        = 32,     
+	eASCII_FORMAT         = 64,     
+	eINCLUDE_FRAME        = 128     
+} TConvertOptions;
 
-//
-// This class represents all information about a channel stored in the DAQ
-// file.
-//
+inline TConvertOptions
+operator+(TConvertOptions left, TConvertOptions right) {
+	return (TConvertOptions)((int)left + (int)right);
+}
+
+inline TConvertOptions
+operator|(TConvertOptions left, TConvertOptions right) {
+	return (TConvertOptions)((int)left | (int)right);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+///
+/// Return code used to specify the status of the end marker of the DAQ file
+///
+typedef enum {
+	eDAQ_EOF_OK = 0,			///< Found full EOF marker
+	eDAQ_EOF_PARTIAL_MARK = 1,	///< Found a partial marker
+	eDAQ_EOF_NOTFOUND = 2,		///< No EOF marker found
+	eDAQ_EOF_UNDEFINED = 3		///< No information available yet
+} TDaqEofStatus;
+
+///////////////////////////////////////////////////////////////////////////////////////////
+///
+/// This class represents all information about a channel stored in the DAQ file. 
+/// Each channel consist of a number of samples, each sample containing one or more 
+/// values of the same type. The term item is used to refer to one of those values. 
+/// The term record is used to refer to all the values in a sample.
+///
+/// The information maintained for each channel includes an identifier (which 
+/// is nothing more than the ordinal number of the channel in the definition array), 
+/// the type of the data, the cardinality of the data, its name and the capture 
+/// rate. The capture rate is given in Hz and is always a sub-multiple of the DAQ 
+/// maximum sampling rate.
+///
+/// In addition, there are three options that refer to the method by which the 
+/// data for a channel is collected. The sampling rate indicates how often 
+/// data for that channel is sampled. Usually, the maximum sampling rate 
+/// is 240 Hz, but most channels are sampled at a frequency lower than that. 
+/// Data stored in 'differential' mode are only stored when their value has 
+/// changed from the previously stored value. This can save a tremendous 
+/// amount of space especially for large but largely unchanged channels. 
+/// Finally, variable length channels are array channels (i.e., cardinality > 1) 
+/// that have only a subset of the array stored for each sample. This is done 
+/// to save space, when the real-time software knows that only a subset of the 
+/// array has valid data. 
+///
 class CDaqChannelInfo {
 public:
 	CDaqChannelInfo() {
@@ -24,7 +98,18 @@ public:
 		m_Name = "";
 	}
 	~CDaqChannelInfo() {};
+    int				GetItemSize()	const;
+	int				GetRecSize()	const;
+	int				GetItemCount()	const;
+	int				GetType()		const;
+	int				GetId()			const;
+	const string&	GetName()		const;
+	int				GetCapRate()	const;
+	bool			IsVarLen()		const;
+
+	friend class CDaqLowLevelIo;
 
+private:
 	int		m_Id;
 	int		m_Items;
 	string	m_Name;
@@ -34,32 +119,345 @@ public:
 };
 
 
+/// Return true, if channel is a variable length array
+inline bool
+CDaqChannelInfo::IsVarLen(void) const
+{
+	return m_VarLen;
+}
+
+
+///// Return sampling rate, in Hz
+inline int
+CDaqChannelInfo::GetCapRate(void) const
+{
+	return m_CapRate;
+}
+
+///// Return name as specified in cell file
+inline const string &
+CDaqChannelInfo::GetName(void) const
+{
+	return m_Name;
+}
+
+/// 
+/// Return the channel identifier, a small integer that can be
+/// used to reference the channel in other calls.
+///
+/// Keep in mind that the channel identifer can change among DAQ files,
+/// even if it refers to the same cell variable.
+///
+inline int
+CDaqChannelInfo::GetId(void) const
+{
+	return m_Id;
+}
+
+///// Return type (c=char, f=float, i=int, s=short, d=double
+inline int
+CDaqChannelInfo::GetType(void) const
+{
+	return m_Type;
+}
+
+///// Return number of items in each sample
+inline int
+CDaqChannelInfo::GetItemCount(void) const
+{
+	return m_Items;
+}
+
+/////////// Return record size, in bytes
+inline int
+CDaqChannelInfo::GetRecSize(void) const
+{
+	return GetItemSize() * GetItemCount(); 
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////////////
+///
+/// This support class provides buffer space for copying daq data out of a 
+/// DAQ file. It is used in lieu of raw buffers to minimize the likelihood of 
+/// memory leaks.
+///
+/// The class itself is not aware of the type or cardinality of the data, 
+/// however, it provides a few type-specific access functions to facilitate 
+/// data access when the user knows the type of data.
+///
+class CDaqBuffer {
+public:
+	CDaqBuffer() { m_Count = 0; };
+	~CDaqBuffer() {};
+
+	vector<TDaqByte>::const_pointer GetDataPtr() const { return &m_Data[0]; };
+
+	int             GetCount() const ///< Return number of typed elements in buffer
+			{ return m_Count; };
+
+	int             GetSize() const ///< Return buffer size in bytes
+			{ return (int)m_Data.size(); };
+
+	char*   GetChar(int ind=0)     ///< Return char type pointer to data
+			{ return ((char *)&m_Data[0]) + ind; };
+
+	int*    GetInt(int ind=0)       ///< Return int type pointer to data
+			{ return ((int *)&m_Data[0]) + ind; };
+
+	float*  GetFloat(int ind=0)     ///< Return float type pointer to data
+			{ return ((float *)&m_Data[0]) + ind; };
+
+	double* GetDouble(int ind=0)    ///< Return double type pointer to data
+			{ return ((double *)&m_Data[0]) + ind; };
+
+	short*  GetShort(int ind=0)     ///< Return short type pointer to data
+			{ return ((short *)&m_Data[0]) + ind; };
+
+	friend class CDaqLowLevelIo;
+	vector<TDaqByte>    m_Data;     // data is stored here
+
+protected:
+	void Append(void *pData, int size, int count);
+	void AllocSpace(int     size, int count=0);
+	int                 m_Count;    // count of data, not in bytes but in type
+};
+
+
+/////////////////////////////////////////////////////////////////////////////////
+///
+/// This function adds data to the DaqBuffer.  It is used by the DAQ library
+/// to build the buffers returned to the user, and as such, is not meant to
+/// be used by the library client.
+///
+/// The function receives a pointer to the data, the size of the buffer in 
+/// bytes and a count of elements in that buffer.  The class deals with
+/// untyped buffers, so the only way to keep track of how many elements
+/// are in the buffer (i.e., 2 integers as opposed to 8 bytes) is for the
+/// caller to provide that information, which in this case is the count 
+/// argument.
+///
+inline void
+CDaqBuffer::Append(
+	void*   p,      ///< pointer to data to append
+	int     size,   ///< size in bytes of data
+	int     count)  ///< count of data items in that buffer
+{
+	size_t oldsize  = m_Data.size();
+
+	m_Data.resize(oldsize + size);
+	memcpy(&m_Data[0]+oldsize, p, size);
+	
+	m_Count += count;
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////
+///
+/// This function allocates space in the DaqBuffer.  
+///
+///
+inline void
+CDaqBuffer::AllocSpace(
+	int     size,   ///< size in bytes of data
+	int     count)  ///< count of data items in that buffer
+{
+	m_Data.resize(size);
+	m_Count = count;
+}
+
+
 /////////////////////////////////////////////////////////////////////////////
-//
-// This class provides an API that does low level reading from a DAQ file
-//
-//
+///
+/// This class provides an API that does low level reading from a DAQ 
+/// file. In particular, each DAQ file is treated as a set of independent 
+/// channels, with each channel having its own type/cardinality/sampling 
+/// rate and sampling method.
+///
+/// As a low level API, it only provides data integrity and raw data 
+/// reading capability for either the whole daq file, or for selected 
+/// channels. Except for these options, there are no other data conditioning 
+/// options or partitioning options. 
+///
 class CDaqLowLevelIo {
 public:
 	CDaqLowLevelIo();
 	~CDaqLowLevelIo();
 
 	bool           Open(const string &);
+	void           GetChannels(TChanInfoVec& channels) const;
+	int            GetFirstFrame() const;
+	int            GetLastFrame() const;  
+	int            GetFrames() const;
+	int            GetFrequency() const;
+	const string&  GetDate() const;
+	const string&  GetVersion() const;
+	const string&  GetTitle() const;
+	const string&  GetSubject() const;
+	const string&  GetRun() const;
+	const string&  GetRunInst() const;
+	const string&  GetFileName() const { return m_Filename; };
+	size_t         GetNumChannels() const;
+	bool           TocFileExists() const { return m_TocExists; };
+
+	bool CheckIntegrity(bool buildToc = false, bool forceFileCreate = false);
+	bool QueryIntegrityValues(TIntVec& drops, TIntVec& dropedFrm, TIntVec& skips, 
+			TDaqEofStatus &stat);
+	const string& GetLastError(void) const 
+			{ return m_LastError; };
+	void Close(void);
+
+	void SetCallbacks( void errf(const char *, const char *), void progf(const char *, int, int));
+
+	bool LowLevelDataRead(const TIntVec* chans, TDataVec& data);
+	bool LowLevelDataCopy(const TIntVec* chans, TFileVec& dest, 
+				TConvertOptions options = eBINARY_FORMAT + eINCLUDE_FRAME);
+	bool GetFullData(const TIntVec* chans, TDataVec& data,  const TIntVec& subsample,
+				TConvertOptions options = eFILL_MISSING_ORDER_0,
+				int frm1 = -1, int frm2 = -1);
 
 private:
+	enum TReadErrorCode {
+		eDAQ_READ_OK,			
+		eDAQ_READ_ERROR,		// read error (short file, premature eof etc.
+		eDAQ_READ_EOF,			// found end marker
+		eDAQ_READ_FAILED,		// lost sync
+		eDAQ_READ_BAD_DATA,		// bad data and couldn't recover
+		eDAQ_NO_SUCH_FRAME,		// asked to read a frame that doesn't exist in TOC
+		eDAQ_READ_INTERNAL_ERROR
+	};
+
+	enum TResyncOutcome {
+		eRESYNC_CHANNEL,	// resynced ok, next thing to read is channel
+		eRESYNC_FRAME,		// resynced ok, next thing to read is frame
+		eRESYNC_NONE		// could not resync
+	};
 
 	bool ReadHeader(FILE *);
 	bool ReadChannelInfo(FILE *);
+	bool DetectFrameHeader(int recFrame);
+	bool ResyncFrame(int curFrame, int& skipCount);
+	TResyncOutcome ResyncChannel(int recFrame, int& skipCount);
+	bool WriteTocFile(const string& fname);
+	bool ReadTocFile(const string& fname);
+	TReadErrorCode ReadFrameHeader(int,	int& ,int&, int&);
+	TReadErrorCode ReadDataForOneFrame(int, TIntVec, TDataVec &, void *);
 
-	string    m_Title;			// As stored in DAQ file
-	string    m_Date;			// As stored in DAQ file
+//	bool GenPurpTraverse(bool frameCB(int frm, int numC, bool end), 
+//							bool channelCB(int frm, int chan) );
+
+	string	m_Filename;
+	string  m_Version;			// file version
+	FILE*	m_pFile;
+	string	m_Title;			// Simulation title, as stored in DAQ file
+	string	m_Date;				// Date of collection, as stored in DAQ file
 	string	m_Subj;				// As stored in DAQ file
 	string	m_Run;				// As stored in DAQ file
 	string	m_RunInst;			// As stored in DAQ file
 	int		m_NumEntries;		// As stored in DAQ file
 	int		m_Frequency;		// As stored in DAQ file
 
-	vector<CDaqChannelInfo>	m_Channels;		// Channel information
+	void	(*m_UserErrorFunc)(const char *, const char *);
+	void	(*m_UserProgrFunc)(const char *, int, int);
+
+	bool                 m_TocExists;	// Indicates a TOC exists in a file
+	bool				 m_HaveToc;		// Indicates if we have TOC in memory
+	TChanInfoVec		 m_Channels;	// Channel information
 	long				 m_DataOffset;	// where data starts in the file 
 	map<int, int>        m_Toc;			// given a sim frame, gives the offset in the file
+	vector<int>	         m_DroppedFrames;
+	int					 m_FirstFrame;
+	int					 m_LastFrame;
+	string				 m_LastError;
+	TDaqEofStatus        m_EofStatus;
+
+	// These are for data integrity
+	TIntVec				m_FrameDrops;		// sequences of skipped frames
+	TIntVec				m_Skips;			// groups of bytes skipped
 };
+
+inline void
+CDaqLowLevelIo::GetChannels(TChanInfoVec & chn) const
+{
+	chn = m_Channels;
+}
+
+/// Return the title stored in the DAQ file
+inline const string &
+CDaqLowLevelIo::GetTitle(void) const
+{
+	return m_Title;
+}
+
+
+/// Return the DAQ file version number
+inline const string &
+CDaqLowLevelIo::GetVersion(void) const
+{
+	return m_Version;
+}
+
+/// Return the run instance name.  If the feature is not
+/// supported due to an older version, an empty string is returned
+inline const string &
+CDaqLowLevelIo::GetRunInst(void) const
+{
+	return m_RunInst;
+}
+
+/// Return the subject name
+inline const string &
+CDaqLowLevelIo::GetSubject(void) const
+{
+	return m_Subj;
+}
+
+/// Return the name of the Run that produced the daq file
+inline const string &
+CDaqLowLevelIo::GetRun(void) const
+{
+	return m_Run;
+}
+
+/// Return the file creation date as stored in the heade
+inline const string &
+CDaqLowLevelIo::GetDate(void) const
+{
+	return m_Date;
+}
+
+
+/// Return the sampling frequency
+inline int
+CDaqLowLevelIo::GetFrequency(void) const
+{
+	return m_Frequency;
+}
+
+/// Return the total number of frames in the file
+inline int
+CDaqLowLevelIo::GetFrames(void) const
+{
+	return m_LastFrame - m_FirstFrame;
+}
+
+/// Return the first frame number
+inline int
+CDaqLowLevelIo::GetFirstFrame(void) const
+{
+	return m_FirstFrame;
+}
+
+/// Return the last frame number
+inline int
+CDaqLowLevelIo::GetLastFrame(void) const
+{
+	return m_LastFrame;
+}
+
+/// This funcion returns the number of channels in the DAQ file
+inline size_t
+CDaqLowLevelIo::GetNumChannels(void) const
+{
+	return m_Channels.size();
+}
\ No newline at end of file