cfl_tokenreader.cpp
Go to the documentation of this file.
1 /** @file cfl_tokenreader.cpp @brief Class TokenReader */
2 
3 /* FAU Discrete Event Systems Library (libfaudes)
4 
5 Copyright (C) 2006 Bernd Opitz
6 Copyright (C) 2006 Thomas Moor
7 Exclusive copyright is granted to Klaus Schmidt
8 
9 This library is free software; you can redistribute it and/or
10 modify it under the terms of the GNU Lesser General Public
11 License as published by the Free Software Foundation; either
12 version 2.1 of the License, or (at your option) any later version.
13 
14 This library is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 Lesser General Public License for more details.
18 
19 You should have received a copy of the GNU Lesser General Public
20 License along with this library; if not, write to the Free Software
21 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
22 
23 
24 
25 
26 #include "cfl_tokenwriter.h"
27 #include "cfl_tokenreader.h"
28 
29 namespace faudes {
30 
31 // TokenReader(mode,instring)
32 TokenReader::TokenReader(Mode mode, const std::string& rInString)
33  : mMode(mode), mpStream(NULL), mFileName("")
34 {
35  switch(mode) {
36  case String:
37  // use mSStream
38  FD_DV("TokenReader::Tokenreader(String, ...): " << rInString);
39  mpSStream= new std::istringstream(rInString, std::istringstream::in | std::istringstream::binary);
41  Rewind();
42  break;
43  case File:
44  // set up mFStream
45  FD_DV("TokenReader::Tokenreader(File, \"" << rInString <<"\")");
46  mFStream.exceptions(std::ios::badbit|std::ios::failbit);
47  try{
48  mFStream.open(rInString.c_str(), std::ios::in | std::ios::binary);
49  }
50  catch (std::ios::failure&) {
51  std::stringstream errstr;
52  errstr << "Exception opening/reading file \""<< rInString << "\"";
53  throw Exception("TokenReader::TokenReader", errstr.str(), 1);
54  }
55  mFileName=rInString;
57  Rewind();
58  break;
59  default:
60  std::stringstream errstr;
61  errstr << "Invalid Mode / Not implemented";
62  throw Exception("TokenReader::TokenReader(mode,instring)", errstr.str(), 1);
63  }
64 }
65 
66 
67 // TokenReader(rFilename)
68 TokenReader::TokenReader(const std::string& rFilename)
69  : mMode(File), mpStream(NULL), mFileName(rFilename)
70 {
71  // set up mFStream
72  FD_DV("TokenReader::Tokenreader(File, \"" << rFilename <<"\")");
73  mFStream.exceptions(std::ios::badbit|std::ios::failbit);
74  try{
75  mFStream.open(rFilename.c_str(), std::ios::in | std::ios::binary);
76  }
77  catch (std::ios::failure&) {
78  std::stringstream errstr;
79  errstr << "Exception opening/reading file \""<< rFilename << "\"";
80  throw Exception("TokenReader::TokenReader", errstr.str(), 1);
81  }
82  mFileName=rFilename;
84  Rewind();
85 }
86 
87 
88 // destruct
90  if(mMode==String) delete mpSStream;
91 }
92 
93 // Stream()
94 std::istream* TokenReader::Streamp(void) {
95  return mpStream;
96 }
97 
98 // Rewind()
99 void TokenReader::Rewind(void) {
100  FD_DV("TokenReader::Rewind: \"" << mFileName <<"\"");
101  try {
102  mpStream->clear();
103  mpStream->seekg(0);
105  mLevel=0;
106  mLevelLabel.clear();
107  mLevelLabel.push_back("STREAM");
108  mLevelPos.clear();
109  mLevelPos.push_back(0);
110  mLevelLine.clear();
111  mLevelLine.push_back(1);
112  mSeekLevel.clear();
113  mLineCount=1;
114  mFilePos=0;
115  }
116  catch (std::ios::failure&) {
117  std::stringstream errstr;
118  errstr << "Exception opening/reading file in "<< FileLine();
119  throw Exception("TokenReader::Rewind", errstr.str(), 1);
120  }
121 }
122 
123 
124 // FileName()
125 std::string TokenReader::FileName(void) const {
126  return mFileName;
127 }
128 
129 // Peek(token)
130 bool TokenReader::Peek(Token& token) {
131  // read to peek buffer
132  if(mPeekToken.IsNone()) {
133  try{
135  } catch (std::ios::failure&) {
136  std::stringstream errstr;
137  errstr << "Exception opening/reading file in "<< FileLine();
138  throw Exception("TokenReader::Peek", errstr.str(), 1);
139  }
140  }
141  // get from peek buffer
142  token=mPeekToken;
143  // substitute empty sections
144  if(token.IsEmpty()) token.ClrEnd();
145  // done
146  FD_DV("TokenReader: Peek: " << token.Str());
147  return !token.IsNone();
148 }
149 
150 // Get(token)
151 bool TokenReader::Get(Token& token) {
152  bool res;
153  // get token from peek buffer
154  res=Peek(token);
155  // invalidate buffer: case a
157  // invalidate buffer: case b
158  if(mPeekToken.IsEmpty()) {
159  FD_DV("TokenReader: fake from : " << mPeekToken.Str());
160  mPeekToken.SetEnd(std::string(mPeekToken.StringValue()));
161  }
162  // bail out n error
163  if(!res) return false;
164  // track state (level of nested sections, filepos etc)
165  if(!token.IsBegin("br") && !token.IsEnd("br")) {
166  mFilePos=mpStream->tellg();
167  if(token.IsBegin()) {
168  mLevel++;
169  mLevelLabel.push_back(token.StringValue());
170  mLevelPos.push_back(mFilePos);
171  mLevelLine.push_back(mLineCount);
172  }
173  if(token.IsEnd()) {
174  mLevel--;
175  if(mLevelLabel.size()<=0 || mLevelPos.size()<=0 || mLevelLine.size()<=0) {
176 #ifdef FAUDES_CHECKED
177  FD_WARN("TokenReader::Get(): Unbalanced end of section \"" << token.StringValue() << "\" at " << FileLine());
178 #endif
179  token.SetNone();
180  return false;
181  }
182 #ifdef FAUDES_CHECKED
183  if(token.StringValue()!=mLevelLabel.back())
184  FD_WARN("TokenReader::Get(): end of section \"" << token.StringValue() << "\" at " << FileLine() << " should match \"" << mLevelLabel.back() << "\" at line " << mLevelLine.back() );
185 #endif
186  mLevelLabel.pop_back();
187  mLevelPos.pop_back();
188  mLevelLine.pop_back();
189  }
190  }
191  FD_DV("TokenReader:Get(): " << token.Str());
192 
193  return res;
194 }
195 
196 // SeekBegin(label)
197 void TokenReader::SeekBegin(const std::string& rLabel) {
198  Token token;
199  SeekBegin(rLabel,token);
200 }
201 
202 // SeekBegin(label)
203 void TokenReader::SeekBegin(const std::string& rLabel, Token& rToken) {
204  // search for begin at any descending level, no rewind
205  FD_DV("TokenReader::SeekBegin: " << rLabel << " at " << FileLine() << " level " << mLevel);
206  int level=mLevel;
207  long int startpos=mFilePos;
208  int startline=mLineCount;
209  for (;;) {
210  // swollow some plain text (e.g. html may contain plain text that cannot be tokenized properly)
212  // exception: did not get a token at all (incl. eof)
213  if(!Peek(rToken)) {
214  Rewind();
215  std::stringstream errstr;
216  errstr << "Section \"" << rLabel << "\" expected at " << FileLine() << " no more tokens";
217  throw Exception("TokenReader::SeekBegin", errstr.str(), 51);
218  }
219  // exception: current section ends
220  if((rToken.Type() == Token::End) && (mLevel == level)) {
221  mpStream->seekg(startpos);
222  mLineCount=startline;
224  std::stringstream errstr;
225  errstr << "Section \"" << rLabel << "\" expected at " << FileLine()
226  << "current section ended unexpected. Found: " << rToken.StringValue() << " Type " << rToken.Type();
227  throw Exception("TokenReader::SeekBegin", errstr.str(), 51);
228  }
229  // success: found begin section
230  if ((rToken.IsBegin()) && (rToken.StringValue() == rLabel))
231  break;
232  // go on seeking
233  Get(rToken);
234  }
235 }
236 
237 // ReadBegin(label)
238 void TokenReader::ReadBegin(const std::string& rLabel) {
239  Token token;
240  ReadBegin(rLabel,token);
241 }
242 
243 // ReadBegin(label,token)
244 void TokenReader::ReadBegin(const std::string& rLabel, Token& rToken) {
245  FD_DV("Looking for Section \"" << rLabel << "\"");
246  try {
247  int level=mLevel;
248  bool firstgo=true;
249  long int startpos=mFilePos;
250  FD_DV("section level " << level << " current pos " << startpos << " begin of section " << mLevelPos[level]);
251  // search for begin at current level
252  for (;;) {
253  // swollow some plain text (e.g. html may contain plain text that cannot be tokenized properly)
255  // exception: did not get a token at all (incl eof)
256  if(!Peek(rToken)) {
257  std::stringstream errstr;
258  errstr << "Section \"" << rLabel << "\" expected at " << FileLine() << ", no token at all";
259  throw Exception("TokenReader::ReadBegin Peek", errstr.str(), 51);
260  }
261  // success: found begin section
262  if((rToken.IsBegin()) && (rToken.StringValue() == rLabel) && (mLevel==level)) {
263  Get(rToken);
264  break;
265  }
266  // rewind once when current section ends
267  if(rToken.IsEnd() && !rToken.IsBegin() && (mLevel == level) && firstgo) {
268  mpStream->seekg(mLevelPos[level]);
269  mFilePos=mLevelPos[level];
270  mLineCount=mLevelLine[level];
271  firstgo=false;
273  continue;
274  }
275  // exception: did not find begin label
276  if((mFilePos>=startpos) && (!firstgo)) {
277  std::stringstream errstr;
278  errstr << "Section \"" << rLabel << "\" expected at " << FileLine() << ", did not find begin label";
279  throw Exception("TokenReader::ReadBegin Missed", errstr.str(), 51);
280  }
281  // skip this token
282  Get(rToken);
283  }
284  }
285  // catch my seek/tell errors
286  catch (std::ios::failure&) {
287  std::stringstream errstr;
288  errstr << "Section \"" << rLabel << "\" expected at " << FileLine();
289  throw Exception("TokenReader::ReadBegin Rewind", errstr.str(), 1);
290  }
291 }
292 
293 
294 // ExistsBegin(label)
295 bool TokenReader::ExistsBegin(const std::string& rLabel) {
296  FD_DV("TokenReader::ExistsBegin(): looking for Section \"" << rLabel << "\"");
297  try {
298  int level=mLevel;
299  bool firstgo=true;
300  long int startpos=mFilePos;
301  FD_DV("section level " << level << " current pos " << startpos << " begin of section " << mLevelPos[level]);
302  Token token;
303  // special case: current is end
304  if(mPeekToken.IsEnd()) return false;
305  // search for begin at current level
306  for(;;) {
307  // swollow some plain text (e.g. html may contain plain text that cannot be tokenized properly)
309  // fail: did not get a token at all (incl eof)
310  if(!Peek(token)) {
311  return false;
312  }
313  // success: found begin section
314  if((token.IsBegin()) && (token.StringValue() == rLabel) && (mLevel==level)) {
315  return true;
316  }
317  // rewind once when current section ends
318  if((token.IsEnd() && !token.IsBegin()) && (mLevel == level) && firstgo) {
319  mpStream->seekg(mLevelPos[level]);
320  mFilePos=mLevelPos[level];
321  mLineCount=mLevelLine[level];
322  firstgo=false;
324  continue;
325  }
326  // fail: did not find begin label
327  if((mFilePos>=startpos) && (!firstgo)) {
328  return false;
329  }
330  // skip this token
331  Get(token);
332  }
333  }
334  // catch my seek/tell errors
335  catch (std::ios::failure&) {
336  std::stringstream errstr;
337  errstr << "Section \"" << rLabel << "\" expected at " << FileLine();
338  throw Exception("TokenReader::ReadBegin Rewind", errstr.str(), 1);
339  }
340  return false;
341 }
342 
343 // ReadEnd(label)
344 void TokenReader::ReadEnd(const std::string& rLabel) {
345  FD_DV("TokenReader::ReadEnd: " << rLabel << " at " << FileLine() );
346  // search for end at current level
347  int level=mLevel;
348  Token token;
349  for (;;) {
350  // swollow some plain text (e.g. html may contain plain text that cannot be tokenized properly)
352  // exception: did not get a token at all
353  if(!Peek(token)) {
354  std::stringstream errstr;
355  errstr << "end of Section \"" << rLabel << "\" expected at " << FileLine();
356  throw Exception("TokenReader::ReadEnd", errstr.str(), 51);
357  }
358  // success: found end of current section
359  if(token.IsEnd() && !token.IsBegin() && (token.StringValue() == rLabel) && (mLevel==level)) {
360  Get(token);
361  break;
362  }
363  // exception: current section ends with unexpected label
364  if(mLevel<level) {
365  std::stringstream errstr;
366  errstr << "end of Section \"" << rLabel << "\" expected at " << FileLine();
367  throw Exception("TokenReader::ReadEnd", errstr.str(), 51);
368  }
369  // get the token and continue
370  Get(token);
371  //std::cout << token.Str() << "\n";
372  }
373 }
374 
375 // Recover()
376 bool TokenReader::Recover(int level) {
377  // trivial cases
378  if(level>mLevel) return false;
379  if(level==mLevel) return true;
380  // loop until match
381  Token token;
382  while(Get(token))
383  if(mLevel<=level) break;
384  // done
385  return level==mLevel;
386 }
387 
388 // Eos(label)
389 bool TokenReader::Eos(const std::string& rLabel) {
390  // peek token and check for end of section
391  Token token;
392  Peek(token);
393  if(! (token.IsEnd() && !token.IsBegin()))
394  return false;
395  if((token.IsEnd() && !token.IsBegin()) && (token.StringValue() == rLabel))
396  return true;
397  std::stringstream errstr;
398  errstr << "Section End\"" << rLabel << "\" expected at " << FileLine();
399  throw Exception("TokenReader::Eos", errstr.str(), 51);
400  return false;
401 }
402 
403 
404 // ReadInteger()
405 long int TokenReader::ReadInteger(void) {
406  Token token;
407  Get(token);
408  if(!token.IsInteger()) {
409  std::stringstream errstr;
410  errstr << "Integer expected at " << FileLine();
411  throw Exception("TokenReader::TokenReader", errstr.str(), 50);
412  }
413  return token.IntegerValue();
414 }
415 
416 // ReadFloat()
418  Token token;
419  Get(token);
420  if((!token.IsFloat()) && (!token.IsInteger())) {
421  std::stringstream errstr;
422  errstr << "Float expected at " << FileLine();
423  throw Exception("TokenReader::TokenReader", errstr.str(), 50);
424  }
425  return token.FloatValue();
426 }
427 
428 // ReadString()
429 const std::string& TokenReader::ReadString(void) {
430  Token token;
431  Get(token);
432  if(!token.IsString()) {
433  std::stringstream errstr;
434  errstr << "Name expected at " << FileLine();
435  throw Exception("TokenReader::TokenReader", errstr.str(), 50);
436  }
437  mLastString=token.StringValue();
438  return(mLastString);
439 }
440 
441 
442 // ReadOption()
443 const std::string& TokenReader::ReadOption(void) {
444  Token token;
445  Get(token);
446  if(!token.IsOption()) {
447  std::stringstream errstr;
448  errstr << "Option expected at " << FileLine();
449  throw Exception("TokenReader::TokenReader", errstr.str(), 50);
450  }
451  mLastString=token.OptionValue();
452  return(mLastString);
453 }
454 
455 // ReadBinary()
456 const std::string& TokenReader::ReadBinary(void) {
457  Token token;
458  Get(token);
459  if(!token.IsBinary()) {
460  std::stringstream errstr;
461  errstr << "Binary string expected at " << FileLine();
462  throw Exception("TokenReader::TokenReader", errstr.str(), 50);
463  }
464  mLastString=token.StringValue();
465  return(mLastString);
466 }
467 
468 
469 // ReadText()
470 const std::string& TokenReader::ReadText(void) {
471  // if we have a markup token we refuse to read text
472  if(mPeekToken.IsBegin() || mPeekToken.IsEnd()) {
473  FD_DV("TokenReader::ReadText(): tag in buffer");
474  mLastString="";
475  return mLastString;
476  }
477  // do my own reading
479  if(ll<0) {
480  std::stringstream errstr;
481  errstr << "Text expected at " << FileLine();
482  throw Exception("TokenReader::TokenReader", errstr.str(), 50);
483  }
484  mLineCount+=ll;
485  // prepend peek buffers string value (better: need rewind!)
486  if(mPeekToken.IsString())
488  // strip leading/tailing spaces
489  static const std::string white=" \n\t\r\v\f";
490  std::size_t pos1=mLastString.find_first_not_of(white);
491  if(pos1!=std::string::npos)
492  mLastString=mLastString.substr(pos1);
493  else
494  mLastString.clear();
495  std::size_t pos2=mLastString.find_last_not_of(white);
496  if(pos2!=std::string::npos)
497  mLastString.erase(pos2+1);
498  // clear peak buffer
500  return mLastString;
501 }
502 
503 // ReadCharacterData()
504 const std::string& TokenReader::ReadCharacterData(void) {
505  // if we have a markup token we refuse to read character data
506  if(mPeekToken.IsBegin() || mPeekToken.IsEnd()) {
507  FD_DV("TokenReader::ReadCharacterData(): tag in buffer");
508  mLastString="";
509  return mLastString;
510  }
511  // do my own reading
513  if(ll<0) {
514  std::stringstream errstr;
515  errstr << "Text expected at " << FileLine();
516  throw Exception("TokenReader::TokenReader", errstr.str(), 50);
517  }
518  mLineCount+=ll;
519  // prepend peek buffers string value (better: need rewind!)
520  if(mPeekToken.IsString())
522  // invalidate buffer
524  return mLastString;
525 }
526 
527 // ReadSection()
528 const std::string& TokenReader::ReadSection(void) {
529  // record current level
530  int clevel = Level();
531  // setup token writer for destination
533  // token copy loop
534  while(true) {
535  // see whether we can grab and copy some character data
536  std::string cdata=ReadCharacterData();
537  tw.WriteCharacterData(cdata);
538  // break end of my level
539  Token token;
540  if(!Peek(token)) break;
541  if(token.IsEnd() && !token.IsBegin() && Level()==clevel)
542  break;
543  // get and copy markup token
544  Get(token);
545  tw.Write(token);
546  }
547  // done
548  mLastString=tw.Str();
549  return mLastString;
550 }
551 
552 
553 // Line()
554 int TokenReader::Line(void) const {
555  return mLineCount;
556 }
557 
558 // FileLine()
559 std::string TokenReader::FileLine(void) const {
560  if(mFileName!="")
561  return "("+ mFileName + ":" + ToStringInteger(mLineCount) +")";
562  else
563  return "(#" + ToStringInteger(mLineCount) +")";
564 }
565 
566 } // namespace faudes

libFAUDES 2.24g --- 2014.09.15 --- c++ api documentaion by doxygen