Latest

DaqFileIO.cxx

@@ -51,7 +51,6 @@ CDaqChannelInfo::GetItemSize() const
 }
 
 
-
 //////////////////////////////////////////////////////////////////////////////////////
 ///
 /// This function allows user to specify error and progress callbacks. The 
@@ -95,6 +94,93 @@ NearDump(FILE* p, int ofs)
  
 /////////////////////////////////////////////////////////////////////////////
 ///
+/// Utility function that writes ASCII data given an untyped pointer to it.  
+///
+///
+static bool 
+WriteAsciiLine(
+	FILE*             pFile, 
+	CDaqChannelInfo&  chInf, 
+	const void*       pData, 
+	TConvertOptions   options)
+{
+	assert(pFile);
+
+	int sample;
+
+	if ( chInf.GetType() == 'c' ) {
+		for (sample=0; sample < chInf.GetItemCount(); sample++) {
+			// skip strings for now
+		}
+	}
+	else if ( chInf.GetType() == 'f' ) {
+		float *p = (float *)pData;
+		for (sample=0; sample < (int)chInf.GetItemCount(); sample++) {
+			fprintf(pFile, "%E ", *p);
+			p++;
+		}
+	}
+	else if ( chInf.GetType() == 'i' ) {
+		int *p = (int *)pData;
+		for (sample=0; sample < (int)chInf.GetItemCount(); sample++) {
+			fprintf(pFile, "%d ", *p);
+			p++;
+		}
+	}
+	else if ( chInf.GetType() == 'd' ) {
+		double *p = (double *)pData;
+		for (sample=0; sample < (int)chInf.GetItemCount(); sample++) {
+			fprintf(pFile, "%E ", *p);
+			p++;
+		}
+	}
+	else if ( chInf.GetType() == 's' ) {
+		int *p = (int *)pData;
+		for (sample=0; sample < (int)chInf.GetItemCount(); sample++) {
+			fprintf(pFile, "%d ", *p);
+			p++;
+		}
+	}
+	else {
+		assert(0);
+	}
+
+	return true;
+}
+
+
+/////////////////////////////////////////////////////////////////////////////
+///
+/// This function determines if a frame was dropped by the DAQ.  The
+/// function requires that integrity testing has taken place (i.e., a TOC
+/// exists.
+///
+bool
+CDaqLowLevelIo::FrameDropped(int frm)
+{
+	vector<int>::const_iterator    pDrpFrm;	// iterates list of dropped frames
+	for (pDrpFrm = m_DroppedFrames.begin(); pDrpFrm != m_DroppedFrames.end(); pDrpFrm++){
+		if ( frm == *pDrpFrm )return true;
+	}
+	return false;
+}
+
+
+/////////////////////////////////////////////////////////////////////////////
+///
+/// This function determines if data was logged for a given frame
+///
+bool
+CDaqLowLevelIo::FrameHasData(int frm)
+{
+	map<int, int>::const_iterator p;
+
+	p = m_Toc.find(frm);
+	return p != m_Toc.end();
+}
+
+/////////////////////////////////////////////////////////////////////////////
+///
 /// Reads the header of the DAQ file, independent of its version, and 
 /// stores the information in variables local to the class instance.  
 /// Values that are not provided in the specific file version are set
@@ -518,7 +604,7 @@ CDaqLowLevelIo::WriteTocFile(
 	if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
 
 	// first frame
-	d1 = this->m_FirstFrame;
+	d1 = m_FirstFrame;
 	if ( fwrite(&d1, sizeof(int), 1, p) != 1 ) return false;
 
 	// last frame
@@ -616,11 +702,12 @@ CDaqLowLevelIo::ReadTocFile(
 	// we don't need the offset now, it's for future compatibility
 	if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
 
+	// Let the TOC contents override auto-calc for first/last frame
 	if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
-	if ( m_FirstFrame != d1 ) return false;
+	m_FirstFrame = d1;
 
 	if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
-	if ( m_LastFrame != d1 ) return false;
+	m_LastFrame = d1;
 
 	if ( fread(&d1, sizeof(int), 1, p) != 1 ) return false;
 	numEntries = d1;
@@ -747,6 +834,7 @@ CDaqLowLevelIo::CheckIntegrity(
 	int		numFrames  = 0;
 	map<int, TFrameInfo>  tempToc;
 	bool    done = false;
+	int     largestFrame = -1;
 
 	//
 	// Go through the file and build the TOC.  Only frames with data that
@@ -768,6 +856,8 @@ CDaqLowLevelIo::CheckIntegrity(
 			if ( (numFrames % 720) == 0 ) (*m_UserProgrFunc)(m_Filename.c_str(), 
 						numFrames, m_LastFrame - m_FirstFrame);
 
+			if ( curFrame > largestFrame ) largestFrame = curFrame;
+
 			frameOffset = ftell(m_pFile) - 12;
 
 			numFrames++;
@@ -892,6 +982,15 @@ CDaqLowLevelIo::CheckIntegrity(
 		int old = -1;
 		int curFrame;
 
+		// We have to be careful in messed up files because the m_LastFrame calculated
+		// by the Open... is the last frame found in the file; In messed up files, the
+		// last frame found is not necessarily the last frame with data
+		if ( largestFrame > m_LastFrame ) {
+			sprintf(buf, "Highest number frame (%d) wasn't last one in file,"
+				"which was %d", largestFrame, m_LastFrame);
+			(*m_UserErrorFunc)(m_Filename.c_str(), buf);
+			m_LastFrame = largestFrame;
+		}
 		while ( frm < m_LastFrame ) {
 			if ( tempToc.find(frm) != tempToc.end() ) {// if it's in the map, it was in the file 
 				curFrame = frm;
@@ -916,6 +1015,7 @@ CDaqLowLevelIo::CheckIntegrity(
 		if ( pTempToc->second.numChan > 0 ) m_Toc[pTempToc->first] = pTempToc->second.ofs;
 	}
 
+
 	m_HaveToc = true;
 
 	if ( writeTocFile ) {
@@ -953,7 +1053,8 @@ CDaqLowLevelIo::CheckIntegrity(
 bool 
 CDaqLowLevelIo::LowLevelDataRead(
 	const vector<int>*  pWhichChannels,  
-	vector<CDaqBuffer>& userStorage
+	vector<CDaqBuffer>& userStorage,
+	vector<TIntVec>*    pFrameList
 )
 {
 	//
@@ -964,7 +1065,12 @@ CDaqLowLevelIo::LowLevelDataRead(
 		return false;
 	}
 
-	if ( pWhichChannels && pWhichChannels->size() != userStorage.size() ) {
+	bool haveChansBadSize;
+	bool noChansBadSize;
+
+	haveChansBadSize = pWhichChannels && pWhichChannels->size() != userStorage.size();
+	noChansBadSize   = pWhichChannels == 0 && userStorage.size() != m_NumEntries;
+	if ( haveChansBadSize || noChansBadSize ) {
 		m_LastError = "inconsistently sized input arguments";
 		return false;
 	}
@@ -989,6 +1095,8 @@ CDaqLowLevelIo::LowLevelDataRead(
 		needChan.resize(m_NumEntries, -1);
 		for (i=0; i<pWhichChannels->size(); i++) {
 			if ( (*pWhichChannels)[i] < 0 || (*pWhichChannels)[i] >= m_NumEntries) {
+				sprintf(buf, "Invalid channel %d", (*pWhichChannels)[i]);
+				m_LastError = buf;
 				return false;
 			}
 			needChan[(*pWhichChannels)[i]] = i;
@@ -1000,16 +1108,23 @@ CDaqLowLevelIo::LowLevelDataRead(
 			needChan.push_back(i);
 	}
 
+	TIntVec empty;
+	if ( pFrameList ) pFrameList->resize(needChan.size(), empty);
 	//
 	// Go through the file collecting the data we need.
 	//
 	int numFrames = 0;
+	int curFrame = -1;     // frame for current record
 	fseek(m_pFile, m_DataOffset, SEEK_SET);
 	while ( 1 ) {
 		int  recHead[3];        // (Code, Frame, Count)
-		int  curFrame = -1;     // frame for current record
 		int  numChan;           // number of channels in current record
 
+		// If we have an end frame, use it as a loop termination criteria
+		if ( m_EofStatus != eDAQ_EOF_NOTFOUND && m_EofStatus != eDAQ_EOF_UNDEFINED ) {
+			if ( curFrame >= m_LastFrame ) break;
+		}
+
 		if ( fread(recHead, sizeof(int), 3, m_pFile) != 3 ) {
 			sprintf(buf, "Couldn't read file, premature end or messed up file");
 			m_LastError = buf;
@@ -1024,12 +1139,11 @@ CDaqLowLevelIo::LowLevelDataRead(
 			// print info every 30 seconds of collection (30*240=720)
 			if ( (numFrames % 720) == 0 ) 
 				(*m_UserProgrFunc)(m_Filename.c_str(), numFrames, m_LastFrame - m_FirstFrame);
-
 		}
 		else if ( recHead[0] == -2 ) {      // This is an end marker
 			curFrame = recHead[1];
 			if ( recHead[2] == DAQ_END_MARK ) {
-					break; 
+				break; // successfully found EOF
 			}
 			else {
 				sprintf(buf, "Unexpected secondary code %d at line %d", recHead[0], 
@@ -1061,11 +1175,17 @@ CDaqLowLevelIo::LowLevelDataRead(
 			int chId;
 
 			if ( fread(&chId, sizeof(int), 1, m_pFile) != 1 ) {
-					sprintf(buf, "Couldn't read file, premature end");
+				sprintf(buf, "Couldn't read file, premature end, frm=%d, ofs=%d", curFrame, ftell(m_pFile));
 				m_LastError = buf;
 				return false;
 			}
-				if ( chId >= m_NumEntries ) {
+			if ( chId >= m_NumEntries || chId < 0 ) {
+				int dummy;
+				int rcode = ResyncChannel(curFrame, dummy);
+				if ( rcode == eRESYNC_CHANNEL || rcode == eRESYNC_FRAME ) {
+					printf("ERROR, ofs %d\n", ftell(m_pFile));
+					break;
+				}
 				sprintf(buf, "Encountered invalid channel %d in data area", chId);
 				m_LastError = buf;
 				return false;
@@ -1090,6 +1210,7 @@ CDaqLowLevelIo::LowLevelDataRead(
 					return false;
 				}
 				daqBuf.Append(pTempSpace, size, ch.GetItemCount());
+				if ( pFrameList ) (*pFrameList)[needChan[chId]].push_back(curFrame);
 			}
 			else {
 				// skip the data
@@ -1104,9 +1225,9 @@ CDaqLowLevelIo::LowLevelDataRead(
 
 ///////////////////////////////////////////////////////////////////////////////////////////
 ///
-/// This function is similar to LowLevelDataRead but instead of reading the data 
-/// into memory buffers it copies the data into a set of output files for which the 
-/// user has already obtained FILE handles.
+/// This function copies  data from a DAQ file into a set of output files for which the 
+/// user has already obtained FILE handles.  The function can optionally fill in
+/// missing data and provides options on how to handle differentially stored data.
 ///
 /// The function will scan the whole file and store the samples of each channel 
 /// specified in chans into the corresponding file. The options argument controls 
@@ -1121,8 +1242,7 @@ CDaqLowLevelIo::LowLevelDataRead(
 /// If the eFILL_MISSING option is set, then the function will detect dropped 
 /// frames and write out replacement values to ensure that all the output file(s) 
 /// contain the appropriate number of samples. Data for dropped frames is generated 
-/// by using the most recent data of the same channel (eFILL_MISSING_ORDER_0), linear 
-/// interpolation (eFILL_MISSING_ORDER_1) or cubic interpolation (eFILL_MISSING_ORDER_2).
+/// by using the most recent data of the same channel.
 ///
 /// The output files can be written in ASCII or binary.  The argument FileHandles
 /// is a vector of FILE pointers that should already have been opened successfully.
@@ -1210,14 +1330,18 @@ CDaqLowLevelIo::LowLevelDataCopy(
 	}
 
 	int    frm = m_FirstFrame;
+	bool   NoExpandDiff = (options & eEXPAND_DIFFERENTIAL) == 0;
+	vector<bool> gotData;
 
+	gotData.resize(m_NumEntries, false);
 	while ( frm <= m_LastFrame ) {
-		map<int, int>::const_iterator  pTocEn;
-		vector<int>::const_iterator    pDrpFrm;
+		map<int, int>::const_iterator  pTocEn;	// iterates list of frames for which we have data
+		vector<int>::const_iterator    pDrpFrm;	// iterates list of dropped frames
 		bool                           curFrameLost = false;
 		int	                           numChan;
 
-		// determine if this is a frame that was dropped
+		// determine if this is a frame that was dropped by searching through the
+		// list of dropped frames calculated during the integrity verification phase
 		for (pDrpFrm = m_DroppedFrames.begin(); pDrpFrm != m_DroppedFrames.end(); pDrpFrm++){
 			if ( frm == *pDrpFrm ) {
 				curFrameLost = true;
@@ -1227,7 +1351,7 @@ CDaqLowLevelIo::LowLevelDataCopy(
 
 		if ( curFrameLost == false ) {
 			pTocEn = m_Toc.find(frm);	// does it have data ?
-			if ( pTocEn == m_Toc.end() ) {	// no data for this frame
+			if ( pTocEn == m_Toc.end() ) {	// no data collected for this frame
 				// do nothing
 			}
 			else {
@@ -1243,32 +1367,79 @@ CDaqLowLevelIo::LowLevelDataCopy(
 				}
 				assert(frm == actualFrameRead);
 
-				ReadDataForOneFrame(numChan, needChan, latestData, pTempSpace);
+				for (int ii=0; ii<m_NumEntries; ii++) gotData[ii] = false;
+				ReadDataForOneFrame(numChan, needChan, latestData, pTempSpace, gotData);
+
 				for (int ch=0; ch < m_NumEntries; ch++) {
 					CDaqChannelInfo& chInf = m_Channels[ch];
 					int              size  = chInf.GetRecSize();
 					size_t           out;
 					
 					int nn = needChan[ch];
+
+					if ( options & eASCII_FORMAT ) {
+						WriteAsciiLine(FileHandles[needChan[ch]],chInf,
+							latestData[needChan[ch]].GetDataPtr(),  options);
+					}
+					else {		// binary format
+						// if this is a differentially stored channel, we must
+						// take into account the user option
+						bool NoNewData    = latestData[needChan[ch]].GetSize() == 0;
+						bool differential = chInf.GetCapRate() < 0;
+
+						if ( differential && NoExpandDiff && NoNewData )
+							continue;
+
+						if ( options & eINCLUDE_FRAME ) {
+							fwrite(&frm, sizeof(int), 1, FileHandles[needChan[ch]]);
+						}
 						out = fwrite(latestData[needChan[ch]].GetDataPtr(), size, 1, 
 									FileHandles[needChan[ch]]);
+						if ( out != 1 ) {
+							sprintf(buf, "Write error while outputing channel %d", ch);
+							m_LastError = buf;
+							return false;
+						}
+					}
 				}
 			}
 		}
 		else {	// current frame lost
+			if ( options & eFILL_MISSING ) {
 				// Produce data for current frame based on prior values
 				for (int ch=0; ch < m_NumEntries; ch++) {
 					CDaqChannelInfo& chInf = m_Channels[ch];
 					int              size  = chInf.GetRecSize();
 					size_t           out;
 
+					bool NoNewData    = latestData[needChan[ch]].GetSize() == 0;
+					bool differential = chInf.GetCapRate() < 0;
+
+					if ( differential && NoExpandDiff && NoNewData )
+						continue;
+
+					if ( options & eASCII_FORMAT ) {
+
+						WriteAsciiLine(FileHandles[needChan[ch]], chInf, 
+							latestData[needChan[ch]].GetDataPtr(), options);
+					}
+					else {
+						if ( options & eINCLUDE_FRAME ) {
+							fwrite(&frm, sizeof(int), 1, FileHandles[needChan[ch]]);
+						}
 						out = fwrite(latestData[needChan[ch]].GetDataPtr(), size, 1, 
 							FileHandles[needChan[ch]]);
+						if ( out != 1 ) {
+							sprintf(buf, "Write error while outputing channel %d", ch);
+							m_LastError = buf;
+							return false;
+						}
+					}
+				}
 			}
 		}
 		frm++;
 	}
-
 	return true;
 }
 
@@ -1279,64 +1450,35 @@ CDaqLowLevelIo::LowLevelDataCopy(
 /// data management and selection of a range of frames to read. Also, unlike 
 /// LowLevelDataRead, this function can only be called if the class has access
 /// to the table of contents (either built with the CheckDataIntegrity function or
-/// read from a prior run). Finally, the function always expands differentially 
-/// stored channels.
+/// read from a prior run). Finally, the function can expand differentially 
+/// stored channels or write them out in compressed format, according to user options
 ///
 /// The function reads the specified list of channels into memory buffers, 
-/// fills in data for dropped frames, can optionally re-sample the data, and 
-/// if specified, read a subset of the data based on a start and end frame.
+/// expands differentially stored data, and fills in data for dropped frames.
+/// If specified, it can read a subset of the data based on a start and end frame.
 ///
 /// The chans vector lists the identifiers for which channels to read. The 
 /// data vector contains data buffers onto which to store the samples of the 
 /// corresponding channels. 
 ///
-/// The subsample vector indicates if the corresponding 
-/// channel should be sub-sampled. The number should be an integer greater than
-/// or equal to 0.  It is interpreted as follows:
-/// If 0, then a data sample is written as often as it was collected in the file
-/// If greater than zero, the function will output data every so many fbs
-/// frames, starting with the first frame on which the channel was collected,
-/// independent of how often data was sampled.  The corresponding entry in the
-/// options array specifies how to interpolate or extrapolate data.
-///
-/// Consider the following example.  The first line indicates during which
-/// fbs frames a channel was collected.  The second line shows the actual
-/// fbs frame numbers.  The remaining lines have an X at each frame at which
-/// data would be output depending on the value of the respective subsample
-/// element.  Note that each time during which the function outputs a value
-/// but there was no corresponding collection at runtime will result in
-/// the application of interpolation according to the options array.
-///
-/// Sample collected :       X           X           X           X
-/// FBS frm          : 0  1  2  3  4  5  6  7  8  9  10 11 12 13 14 15
-/// subsample = 0    :       X           X           X           X
-/// subsample = 1    :       X  X  X  X  X  X  X  X  X  X  X  X  X  X
-/// subsample = 2    :       X     X     X     X     X     X     X
-/// subsample = 3    :       X        X        X        X        X
-/// subsample = 4    :       X           X           X           X
-///
 /// Finally, note that for differentially stored data, the extrapolation 
-/// method defaults to zero-hold, no matter what specification is given.
-///
-/// The options argument is used to specify how to deal with dropped frames 
-/// and as such, the function is only concerned with the eFILL_MISSING_XXXXXX 
-/// options. In particular, specifying eFILL_MISSING will fill in dropped 
-/// frames by using a zero-hold technique by default. Specify any 
-/// of eFILL_MISSING_ORDER_? constants to modify the default behavior. 
+/// method defaults to zero-hold.
 ///
 /// The frm1 and frm2 arguments specify the first/last frame to read, inclusively.
-/// Specify -1 for either, to read from the start or to the end of file respectively.
+/// Specify -1 to read from the start or to the end of file respectively.
 /// The frame numbers are 0 based, so the first frame would be 0.
 ///
 bool 
 CDaqLowLevelIo::GetFullData(
 	const TIntVec*  pWhichChannels,      
-	TDataVec&       data,       
-	const TIntVec&  subsample,  
-	TConvertOptions options,    
-	int             frm1,       
-	int             frm2)       
+	TDataVec&       userDataStorage,	///< Where to store the data
+	TConvertOptions option,
+	int             frm1,				///< First frame to sample, or -1 for first in file
+	int             frm2,				///< Last frame to sample, or -1 for last in file
+	vector<TIntVec>* pFrmList)			///< Where to store frame sequences
 {
+	char buf[256];						// buffer for writing messages
+
 	//
 	// Standard error checking
 	//
@@ -1344,19 +1486,18 @@ CDaqLowLevelIo::GetFullData(
 		m_LastError = "uninitialized class instance";
 		return false;
 	}
-
+	if ( m_HaveToc == false ) {
+		m_LastError = "need to buld table of contents first";
+		return false;
+	}
 	if ( frm1 > GetFrames() || frm2 > GetFrames() ) {
 		m_LastError = "invalid frame range";
 		return false;
 	}
-
-	// 
-	// Adjust frame numbers so they reflect fbs frames
-	//
-	if ( frm1 < 0 ) frm1 = GetFirstFrame(); else frm1 += GetFirstFrame();
-	if ( frm2 < 0 ) frm2 = GetLastFrame(); else frm2 += GetLastFrame();
-
-	char buf[256];              // buffer for writing messages
+	if ( (option & eINCLUDE_FRAME) && pFrmList == 0 ) {
+		m_LastError = "must provide storage for frames when eINCLUDE_FRAME is specified";
+		return false;
+	}
 
 	//
 	// Create a faster data structure to tell us which channels are needed.
@@ -1365,21 +1506,35 @@ CDaqLowLevelIo::GetFullData(
 	//
 	vector<int>  needChan;
 	if ( pWhichChannels ) {
-		unsigned     i;
-
 		needChan.resize(m_NumEntries, -1);
-		for (i=0; i<pWhichChannels->size(); i++) {
+		for (unsigned i=0; i<pWhichChannels->size(); i++) {
 			if ( (*pWhichChannels)[i] < 0 || (*pWhichChannels)[i] >= m_NumEntries) {
+				sprintf(buf, "invalid channel id %d ", (*pWhichChannels)[i]);
+				m_LastError = buf;
 				return false;
 			}
 			needChan[(*pWhichChannels)[i]] = i;
 		}
 	}
 	else {
-		int i;
-		for (i=0; i<m_NumEntries; i++)
-			needChan.push_back(i);
+		for (int i=0; i<m_NumEntries; i++) needChan.push_back(i);
+	}
+
+	vector<bool>        gotData;					// keep track of which channel has data
+	const vector<int>*  pChnList;
+	vector<int>         fullList;
+
+	gotData.resize(m_NumEntries, false);
+
+	if ( pWhichChannels ) {
+		pChnList = pWhichChannels;
 	}
+	else {
+		fullList.resize(m_NumEntries);
+		for (int idx=0; idx < m_NumEntries; idx++) fullList[idx] = idx;
+		pChnList = &fullList;
+	}
+
 	// temporary storage; use a vector so we don't have to worry about
 	// memory leaks
 	vector<unsigned char>  tempSpace;
@@ -1390,35 +1545,111 @@ CDaqLowLevelIo::GetFullData(
 	// Various other variables
 	//
 	vector<CDaqBuffer>   latestData;		// latest sample of data
-	vector<int>          notNeededNow;		// helps maintain when a sample is needed
-	vector<bool>         latestFromThisFrm;	// when true, latestData defined on this frame
-//	int                  startOfs;
-	map<int, int>::const_iterator	mapIter;
+	CDaqBuffer           temBuf;			// empty buffer to be used as template for latestData
 	int                  curFrame;
 	int                  numChan;           // number of channels in current record
 	int                  bytesSkipped;
+	TIntVec              empty;
 
+	if ( pFrmList) pFrmList->resize(userDataStorage.size(), empty);
+	latestData.resize(pChnList->size(), temBuf);
 	CDaqLowLevelIo::TReadErrorCode rcode;
 
+	// 
+	// Adjust frame numbers so they reflect fbs frames
+	//
+	if ( frm1 < 0 ) frm1 = GetFirstFrame(); else frm1 += GetFirstFrame();
+	if ( frm2 < 0 ) frm2 = GetLastFrame(); else frm2 += GetLastFrame();
+
+	//
+	// If the first frame to read is not the first one in the file, and if
+	// there are differential channels, we must find the most recent value
+	// of all differential channels, otherwise we may need to output data nad
+	// have none
+	//
+	if ( frm1 == GetFirstFrame() ) {
+		for (size_t idx=0; idx < pChnList->size(); idx++) {
+			if ( m_Channels[(*pChnList)[idx]].GetCapRate() < 0 ) {
+				// here search backwards from frame frm1 until
+				// we find data for the specific channel.  Store
+				// its values in latestData[idx]
+			}
+		}
+	}
+
+	// Seek to the first frame we need
 	rcode = ReadFrameHeader(frm1, curFrame, numChan, bytesSkipped);
 	if ( rcode != eDAQ_READ_OK ) {
-		sprintf(buf, "ReadFrameHeader returned %d",  rcode);
+		sprintf(buf, "First ReadFrameHeader() returned %d",  rcode);
 		m_LastError = buf;
 		return false;
 	}
 
-	// -> Read data into latestData, make sure we have read one of everything
+	while ( curFrame < frm2 ) {
+		bool curFrameLost = FrameDropped(curFrame);
 
-	while ( curFrame <= frm2 ) {
-		size_t chan;
+		if ( curFrameLost ) {
+			if ( option & eFILL_MISSING ) {
+				// Re-use the most recent data
+				for (size_t idx=0; idx < pChnList->size(); idx++) {
+					CDaqChannelInfo& chInf = m_Channels[(*pChnList)[idx]];
+					int              size  = chInf.GetRecSize();
+
+					userDataStorage[idx].Append(latestData[idx].GetDataPtr(), size,
+						chInf.GetItemCount());
+					if ( pFrmList ) (*pFrmList)[idx].push_back(curFrame);
+				}
+			}
+		}
+		else {
+			if ( numChan > 0 ) {
+				for (int ii=0; ii < m_NumEntries; ii++) gotData[ii] = false;
 
-		for (chan = 0; chan < m_Channels.size(); chan++) {
-			ReadDataForOneFrame(numChan, needChan, latestData, pTempSpace);
+				ReadDataForOneFrame(numChan, needChan, latestData, pTempSpace, gotData);
+				for (size_t idx=0; idx < pChnList->size(); idx++) {
+					CDaqChannelInfo& chInf    = m_Channels[(*pChnList)[idx]];
+					int              size     = chInf.GetRecSize();
+					bool             diffChan = chInf.GetCapRate() < 0;
+					const void*      pSrc = latestData[idx].GetDataPtr();
 
+					if ( diffChan && (option & eEXPAND_DIFFERENTIAL) ) {
+						userDataStorage[idx].Append(pSrc, size, chInf.GetItemCount());
+						if ( pFrmList) (*pFrmList)[idx].push_back(curFrame);
+					}
+					else if ( !diffChan && gotData[(*pChnList)[idx]] ) {
+						userDataStorage[idx].Append(pSrc, size, chInf.GetItemCount());
+						if ( pFrmList) (*pFrmList)[idx].push_back(curFrame);
 					}
 				}
+			}
+			else {		// no data for that frame
+				if ( option & eEXPAND_DIFFERENTIAL ) {
+					// when expanding differential data, we output samples for each frame
+					for (size_t idx=0; idx < pChnList->size(); idx++) {
+						int chId = (*pChnList)[idx];
 
+						if ( m_Channels[chId].GetCapRate() < 0 ) {
+							int         size = m_Channels[chId].GetRecSize();
+							const void* pSrc = latestData[idx].GetDataPtr();
+							userDataStorage[idx].Append(pSrc, size, m_Channels[chId].GetItemCount());
+							if ( pFrmList ) (*pFrmList) [idx].push_back(curFrame);
+						}
+					}
+				}
+				else {
+                    // Do nothing
+				}
+			}
+		}
+		rcode = ReadFrameHeader(-1, curFrame, numChan, bytesSkipped);
+		if ( rcode != eDAQ_READ_OK && rcode != eDAQ_READ_EOF) {
+			sprintf(buf, "ReadFrameHeader returned %d (curFrame=%d, last=%d)",  rcode, curFrame, frm2);
+			m_LastError = buf;
 			return false;
+		}
+	}
+
+	return true;
 }
 
 
@@ -1454,7 +1685,8 @@ CDaqLowLevelIo::ReadDataForOneFrame(
 	int           numEntriesInThisRecord,
 	vector<int>   needChannel,
 	TDataVec&     storage,
-	void*         pTempSpace
+	void*         pTempSpace,
+	vector<bool>& gotData
 )
 {
 	int idx;
@@ -1464,7 +1696,7 @@ CDaqLowLevelIo::ReadDataForOneFrame(
 		int chId;
 		if ( fread(&chId, sizeof(int), 1, m_pFile) != 1 ) return eDAQ_READ_ERROR;
 
-		if ( chId >= m_NumEntries ) {
+		if ( chId >= m_NumEntries || chId < 0 ) {
 			sprintf(buf, "Encountered invalid channel %d in data area", chId);
 			m_LastError = buf;
 			return eDAQ_READ_BAD_DATA;
@@ -1475,7 +1707,6 @@ CDaqLowLevelIo::ReadDataForOneFrame(
 		if ( needChannel[chId] != -1 ) {	// it is -1 when we don't need it
 			CDaqBuffer&  daqBuf = storage[needChannel[chId]];
 
-			daqBuf.AllocSpace(0);		// clear contents
 			if ( size > cMaxDaqRec ) {
 				sprintf(buf, "Daq record size too big, increase \"cMaxDaqRec\"");
 				m_LastError = buf;
@@ -1488,7 +1719,8 @@ CDaqLowLevelIo::ReadDataForOneFrame(
 				m_LastError = buf;
 				return eDAQ_READ_ERROR;
 			}
-			daqBuf.Append(pTempSpace, size, ch.GetItemCount());
+			daqBuf.Replace(pTempSpace, size, ch.GetItemCount());
+			gotData[chId] = true;
 		}
 		else {
 			// skip the data