cfl_tokenreader.cppGo to the documentation of this file.00001 /** @file cfl_tokenreader.cpp @brief Class TokenReader */ 00002 00003 /* FAU Discrete Event Systems Library (libfaudes) 00004 00005 Copyright (C) 2006 Bernd Opitz 00006 Copyright (C) 2006 Thomas Moor 00007 Exclusive copyright is granted to Klaus Schmidt 00008 00009 This library is free software; you can redistribute it and/or 00010 modify it under the terms of the GNU Lesser General Public 00011 License as published by the Free Software Foundation; either 00012 version 2.1 of the License, or (at your option) any later version. 00013 00014 This library is distributed in the hope that it will be useful, 00015 but WITHOUT ANY WARRANTY; without even the implied warranty of 00016 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 00017 Lesser General Public License for more details. 00018 00019 You should have received a copy of the GNU Lesser General Public 00020 License along with this library; if not, write to the Free Software 00021 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ 00022 00023 00024 00025 00026 #include "cfl_tokenwriter.h" 00027 #include "cfl_tokenreader.h" 00028 00029 namespace faudes { 00030 00031 // TokenReader(mode,instring) 00032 TokenReader::TokenReader(Mode mode, const std::string& rInString) 00033 : mMode(mode), mpStream(NULL), mFileName("") 00034 { 00035 switch(mode) { 00036 case String: 00037 // use mSStream 00038 FD_DV("TokenReader::Tokenreader(String, ...): " << rInString); 00039 mpSStream= new std::istringstream(rInString, std::istringstream::in | std::istringstream::binary); 00040 mpStream= mpSStream; 00041 Rewind(); 00042 break; 00043 case File: 00044 // set up mFStream 00045 FD_DV("TokenReader::Tokenreader(File, \"" << rInString <<"\")"); 00046 mFStream.exceptions(std::ios::badbit|std::ios::failbit); 00047 try{ 00048 mFStream.open(rInString.c_str(), std::ios::in | std::ios::binary); 00049 } 00050 catch (std::ios::failure&) { 00051 std::stringstream errstr; 00052 errstr << "Exception opening/reading file \""<< rInString << "\""; 00053 throw Exception("TokenReader::TokenReader", errstr.str(), 1); 00054 } 00055 mFileName=rInString; 00056 mpStream=&mFStream; 00057 Rewind(); 00058 break; 00059 default: 00060 std::stringstream errstr; 00061 errstr << "Invalid Mode / Not implemented"; 00062 throw Exception("TokenReader::TokenReader(mode,instring)", errstr.str(), 1); 00063 } 00064 } 00065 00066 00067 // TokenReader(rFilename) 00068 TokenReader::TokenReader(const std::string& rFilename) 00069 : mMode(File), mpStream(NULL), mFileName(rFilename) 00070 { 00071 // set up mFStream 00072 FD_DV("TokenReader::Tokenreader(File, \"" << rFilename <<"\")"); 00073 mFStream.exceptions(std::ios::badbit|std::ios::failbit); 00074 try{ 00075 mFStream.open(rFilename.c_str(), std::ios::in | std::ios::binary); 00076 } 00077 catch (std::ios::failure&) { 00078 std::stringstream errstr; 00079 errstr << "Exception opening/reading file \""<< rFilename << "\""; 00080 throw Exception("TokenReader::TokenReader", errstr.str(), 1); 00081 } 00082 mFileName=rFilename; 00083 mpStream=&mFStream; 00084 Rewind(); 00085 } 00086 00087 00088 // destruct 00089 TokenReader::~TokenReader(void) { 00090 if(mMode==String) delete mpSStream; 00091 } 00092 00093 // Stream() 00094 std::istream* TokenReader::Streamp(void) { 00095 return mpStream; 00096 } 00097 00098 // Rewind() 00099 void TokenReader::Rewind(void) { 00100 FD_DV("TokenReader::Rewind: \"" << mFileName <<"\""); 00101 try { 00102 mpStream->clear(); 00103 mpStream->seekg(0); 00104 mPeekToken.SetNone(); 00105 mLevel=0; 00106 mLevelLabel.clear(); 00107 mLevelLabel.push_back("STREAM"); 00108 mLevelPos.clear(); 00109 mLevelPos.push_back(0); 00110 mLevelLine.clear(); 00111 mLevelLine.push_back(1); 00112 mSeekLevel.clear(); 00113 mLineCount=1; 00114 mFilePos=0; 00115 } 00116 catch (std::ios::failure&) { 00117 std::stringstream errstr; 00118 errstr << "Exception opening/reading file in "<< FileLine(); 00119 throw Exception("TokenReader::Rewind", errstr.str(), 1); 00120 } 00121 } 00122 00123 00124 // FileName() 00125 std::string TokenReader::FileName(void) const { 00126 return mFileName; 00127 } 00128 00129 // Peek(token) 00130 bool TokenReader::Peek(Token& token) { 00131 // read to peek buffer 00132 if(mPeekToken.IsNone()) { 00133 try{ 00134 mLineCount += mPeekToken.Read(mpStream); 00135 } catch (std::ios::failure&) { 00136 std::stringstream errstr; 00137 errstr << "Exception opening/reading file in "<< FileLine(); 00138 throw Exception("TokenReader::Peek", errstr.str(), 1); 00139 } 00140 } 00141 // get from peek buffer 00142 token=mPeekToken; 00143 // substitute empty sections 00144 if(token.IsEmpty()) token.ClrEnd(); 00145 // done 00146 FD_DV("TokenReader: Peek: " << token.Str()); 00147 return !token.IsNone(); 00148 } 00149 00150 // Get(token) 00151 bool TokenReader::Get(Token& token) { 00152 bool res; 00153 // get token from peek buffer 00154 res=Peek(token); 00155 // invalidate buffer: case a 00156 if(!mPeekToken.IsEmpty()) mPeekToken.SetNone(); 00157 // invalidate buffer: case b 00158 if(mPeekToken.IsEmpty()) { 00159 FD_DV("TokenReader: fake from : " << mPeekToken.Str()); 00160 mPeekToken.SetEnd(std::string(mPeekToken.StringValue())); 00161 } 00162 // bail out n error 00163 if(!res) return false; 00164 // track state (level of nested sections, filepos etc) 00165 if(!token.IsBegin("br") && !token.IsEnd("br")) { 00166 mFilePos=mpStream->tellg(); 00167 if(token.IsBegin()) { 00168 mLevel++; 00169 mLevelLabel.push_back(token.StringValue()); 00170 mLevelPos.push_back(mFilePos); 00171 mLevelLine.push_back(mLineCount); 00172 } 00173 if(token.IsEnd()) { 00174 mLevel--; 00175 if(mLevelLabel.size()<=0 || mLevelPos.size()<=0 || mLevelLine.size()<=0) { 00176 #ifdef FAUDES_CHECKED 00177 FD_WARN("TokenReader::Get(): Unbalanced end of section \"" << token.StringValue() << "\" at " << FileLine()); 00178 #endif 00179 token.SetNone(); 00180 return false; 00181 } 00182 #ifdef FAUDES_CHECKED 00183 if(token.StringValue()!=mLevelLabel.back()) 00184 FD_WARN("TokenReader::Get(): end of section \"" << token.StringValue() << "\" at " << FileLine() << " should match \"" << mLevelLabel.back() << "\" at line " << mLevelLine.back() ); 00185 #endif 00186 mLevelLabel.pop_back(); 00187 mLevelPos.pop_back(); 00188 mLevelLine.pop_back(); 00189 } 00190 } 00191 FD_DV("TokenReader:Get(): " << token.Str()); 00192 00193 return res; 00194 } 00195 00196 // SeekBegin(label) 00197 void TokenReader::SeekBegin(const std::string& rLabel) { 00198 Token token; 00199 SeekBegin(rLabel,token); 00200 } 00201 00202 // SeekBegin(label) 00203 void TokenReader::SeekBegin(const std::string& rLabel, Token& rToken) { 00204 // search for begin at any descending level, no rewind 00205 FD_DV("TokenReader::SeekBegin: " << rLabel << " at " << FileLine() << " level " << mLevel); 00206 int level=mLevel; 00207 long int startpos=mFilePos; 00208 int startline=mLineCount; 00209 for (;;) { 00210 // swollow some plain text (e.g. html may contain plain text that cannot be tokenized properly) 00211 ReadCharacterData(); 00212 // exception: did not get a token at all (incl. eof) 00213 if(!Peek(rToken)) { 00214 Rewind(); 00215 std::stringstream errstr; 00216 errstr << "Section \"" << rLabel << "\" expected at " << FileLine() << " no more tokens"; 00217 throw Exception("TokenReader::SeekBegin", errstr.str(), 51); 00218 } 00219 // exception: current section ends 00220 if((rToken.Type() == Token::End) && (mLevel == level)) { 00221 mpStream->seekg(startpos); 00222 mLineCount=startline; 00223 mPeekToken.SetNone(); 00224 std::stringstream errstr; 00225 errstr << "Section \"" << rLabel << "\" expected at " << FileLine() 00226 << "current section ended unexpected. Found: " << rToken.StringValue() << " Type " << rToken.Type(); 00227 throw Exception("TokenReader::SeekBegin", errstr.str(), 51); 00228 } 00229 // success: found begin section 00230 if ((rToken.IsBegin()) && (rToken.StringValue() == rLabel)) 00231 break; 00232 // go on seeking 00233 Get(rToken); 00234 } 00235 } 00236 00237 // ReadBegin(label) 00238 void TokenReader::ReadBegin(const std::string& rLabel) { 00239 Token token; 00240 ReadBegin(rLabel,token); 00241 } 00242 00243 // ReadBegin(label,token) 00244 void TokenReader::ReadBegin(const std::string& rLabel, Token& rToken) { 00245 FD_DV("Looking for Section \"" << rLabel << "\""); 00246 try { 00247 int level=mLevel; 00248 bool firstgo=true; 00249 long int startpos=mFilePos; 00250 FD_DV("section level " << level << " current pos " << startpos << " begin of section " << mLevelPos[level]); 00251 // search for begin at current level 00252 for (;;) { 00253 // swollow some plain text (e.g. html may contain plain text that cannot be tokenized properly) 00254 ReadCharacterData(); 00255 // exception: did not get a token at all (incl eof) 00256 if(!Peek(rToken)) { 00257 std::stringstream errstr; 00258 errstr << "Section \"" << rLabel << "\" expected at " << FileLine() << ", no token at all"; 00259 throw Exception("TokenReader::ReadBegin Peek", errstr.str(), 51); 00260 } 00261 // success: found begin section 00262 if((rToken.IsBegin()) && (rToken.StringValue() == rLabel) && (mLevel==level)) { 00263 Get(rToken); 00264 break; 00265 } 00266 // rewind once when current section ends 00267 if(rToken.IsEnd() && !rToken.IsBegin() && (mLevel == level) && firstgo) { 00268 mpStream->seekg(mLevelPos[level]); 00269 mFilePos=mLevelPos[level]; 00270 mLineCount=mLevelLine[level]; 00271 firstgo=false; 00272 mPeekToken.SetNone(); 00273 continue; 00274 } 00275 // exception: did not find begin label 00276 if((mFilePos>=startpos) && (!firstgo)) { 00277 std::stringstream errstr; 00278 errstr << "Section \"" << rLabel << "\" expected at " << FileLine() << ", did not find begin label"; 00279 throw Exception("TokenReader::ReadBegin Missed", errstr.str(), 51); 00280 } 00281 // skip this token 00282 Get(rToken); 00283 } 00284 } 00285 // catch my seek/tell errors 00286 catch (std::ios::failure&) { 00287 std::stringstream errstr; 00288 errstr << "Section \"" << rLabel << "\" expected at " << FileLine(); 00289 throw Exception("TokenReader::ReadBegin Rewind", errstr.str(), 1); 00290 } 00291 } 00292 00293 00294 // ExistsBegin(label) 00295 bool TokenReader::ExistsBegin(const std::string& rLabel) { 00296 FD_DV("TokenReader::ExistsBegin(): looking for Section \"" << rLabel << "\""); 00297 try { 00298 int level=mLevel; 00299 bool firstgo=true; 00300 long int startpos=mFilePos; 00301 FD_DV("section level " << level << " current pos " << startpos << " begin of section " << mLevelPos[level]); 00302 Token token; 00303 // special case: current is end 00304 if(mPeekToken.IsEnd()) return false; 00305 // search for begin at current level 00306 for(;;) { 00307 // swollow some plain text (e.g. html may contain plain text that cannot be tokenized properly) 00308 ReadCharacterData(); 00309 // fail: did not get a token at all (incl eof) 00310 if(!Peek(token)) { 00311 return false; 00312 } 00313 // success: found begin section 00314 if((token.IsBegin()) && (token.StringValue() == rLabel) && (mLevel==level)) { 00315 return true; 00316 } 00317 // rewind once when current section ends 00318 if((token.IsEnd() && !token.IsBegin()) && (mLevel == level) && firstgo) { 00319 mpStream->seekg(mLevelPos[level]); 00320 mFilePos=mLevelPos[level]; 00321 mLineCount=mLevelLine[level]; 00322 firstgo=false; 00323 mPeekToken.SetNone(); 00324 continue; 00325 } 00326 // fail: did not find begin label 00327 if((mFilePos>=startpos) && (!firstgo)) { 00328 return false; 00329 } 00330 // skip this token 00331 Get(token); 00332 } 00333 } 00334 // catch my seek/tell errors 00335 catch (std::ios::failure&) { 00336 std::stringstream errstr; 00337 errstr << "Section \"" << rLabel << "\" expected at " << FileLine(); 00338 throw Exception("TokenReader::ReadBegin Rewind", errstr.str(), 1); 00339 } 00340 return false; 00341 } 00342 00343 // ReadEnd(label) 00344 void TokenReader::ReadEnd(const std::string& rLabel) { 00345 FD_DV("TokenReader::ReadEnd: " << rLabel << " at " << FileLine() ); 00346 // search for end at current level 00347 int level=mLevel; 00348 Token token; 00349 for (;;) { 00350 // swollow some plain text (e.g. html may contain plain text that cannot be tokenized properly) 00351 ReadCharacterData(); 00352 // exception: did not get a token at all 00353 if(!Peek(token)) { 00354 std::stringstream errstr; 00355 errstr << "end of Section \"" << rLabel << "\" expected at " << FileLine(); 00356 throw Exception("TokenReader::ReadEnd", errstr.str(), 51); 00357 } 00358 // success: found end of current section 00359 if(token.IsEnd() && !token.IsBegin() && (token.StringValue() == rLabel) && (mLevel==level)) { 00360 Get(token); 00361 break; 00362 } 00363 // exception: current section ends with unexpected label 00364 if(mLevel<level) { 00365 std::stringstream errstr; 00366 errstr << "end of Section \"" << rLabel << "\" expected at " << FileLine(); 00367 throw Exception("TokenReader::ReadEnd", errstr.str(), 51); 00368 } 00369 // get the token and continue 00370 Get(token); 00371 //std::cout << token.Str() << "\n"; 00372 } 00373 } 00374 00375 // Recover() 00376 bool TokenReader::Recover(int level) { 00377 // trivial cases 00378 if(level>mLevel) return false; 00379 if(level==mLevel) return true; 00380 // loop until match 00381 Token token; 00382 while(Get(token)) 00383 if(mLevel<=level) break; 00384 // done 00385 return level==mLevel; 00386 } 00387 00388 // Eos(label) 00389 bool TokenReader::Eos(const std::string& rLabel) { 00390 // peek token and check for end of section 00391 Token token; 00392 Peek(token); 00393 if(! (token.IsEnd() && !token.IsBegin())) 00394 return false; 00395 if((token.IsEnd() && !token.IsBegin()) && (token.StringValue() == rLabel)) 00396 return true; 00397 std::stringstream errstr; 00398 errstr << "Section End\"" << rLabel << "\" expected at " << FileLine(); 00399 throw Exception("TokenReader::Eos", errstr.str(), 51); 00400 return false; 00401 } 00402 00403 00404 // ReadInteger() 00405 long int TokenReader::ReadInteger(void) { 00406 Token token; 00407 Get(token); 00408 if(!token.IsInteger()) { 00409 std::stringstream errstr; 00410 errstr << "Integer expected at " << FileLine(); 00411 throw Exception("TokenReader::TokenReader", errstr.str(), 50); 00412 } 00413 return token.IntegerValue(); 00414 } 00415 00416 // ReadFloat() 00417 double TokenReader::ReadFloat(void) { 00418 Token token; 00419 Get(token); 00420 if((!token.IsFloat()) && (!token.IsInteger())) { 00421 std::stringstream errstr; 00422 errstr << "Float expected at " << FileLine(); 00423 throw Exception("TokenReader::TokenReader", errstr.str(), 50); 00424 } 00425 return token.FloatValue(); 00426 } 00427 00428 // ReadString() 00429 const std::string& TokenReader::ReadString(void) { 00430 Token token; 00431 Get(token); 00432 if(!token.IsString()) { 00433 std::stringstream errstr; 00434 errstr << "Name expected at " << FileLine(); 00435 throw Exception("TokenReader::TokenReader", errstr.str(), 50); 00436 } 00437 mLastString=token.StringValue(); 00438 return(mLastString); 00439 } 00440 00441 00442 // ReadOption() 00443 const std::string& TokenReader::ReadOption(void) { 00444 Token token; 00445 Get(token); 00446 if(!token.IsOption()) { 00447 std::stringstream errstr; 00448 errstr << "Option expected at " << FileLine(); 00449 throw Exception("TokenReader::TokenReader", errstr.str(), 50); 00450 } 00451 mLastString=token.OptionValue(); 00452 return(mLastString); 00453 } 00454 00455 // ReadBinary() 00456 const std::string& TokenReader::ReadBinary(void) { 00457 Token token; 00458 Get(token); 00459 if(!token.IsBinary()) { 00460 std::stringstream errstr; 00461 errstr << "Binary string expected at " << FileLine(); 00462 throw Exception("TokenReader::TokenReader", errstr.str(), 50); 00463 } 00464 mLastString=token.StringValue(); 00465 return(mLastString); 00466 } 00467 00468 00469 // ReadText() 00470 const std::string& TokenReader::ReadText(void) { 00471 // if we have a markup token we refuse to read text 00472 if(mPeekToken.IsBegin() || mPeekToken.IsEnd()) { 00473 FD_DV("TokenReader::ReadText(): tag in buffer"); 00474 mLastString=""; 00475 return mLastString; 00476 } 00477 // do my own reading 00478 int ll=Token::ReadEscapedString(mpStream,'<',mLastString); 00479 if(ll<0) { 00480 std::stringstream errstr; 00481 errstr << "Text expected at " << FileLine(); 00482 throw Exception("TokenReader::TokenReader", errstr.str(), 50); 00483 } 00484 mLineCount+=ll; 00485 // prepend peek buffers string value (better: need rewind!) 00486 if(mPeekToken.IsString()) 00487 mLastString=mPeekToken.StringValue() + " " + mLastString; 00488 // strip leading/tailing spaces 00489 static const std::string white=" \n\t\r\v\f"; 00490 std::size_t pos1=mLastString.find_first_not_of(white); 00491 if(pos1!=std::string::npos) 00492 mLastString=mLastString.substr(pos1); 00493 else 00494 mLastString.clear(); 00495 std::size_t pos2=mLastString.find_last_not_of(white); 00496 if(pos2!=std::string::npos) 00497 mLastString.erase(pos2+1); 00498 // clear peak buffer 00499 mPeekToken.SetNone(); 00500 return mLastString; 00501 } 00502 00503 // ReadCharacterData() 00504 const std::string& TokenReader::ReadCharacterData(void) { 00505 // if we have a markup token we refuse to read character data 00506 if(mPeekToken.IsBegin() || mPeekToken.IsEnd()) { 00507 FD_DV("TokenReader::ReadCharacterData(): tag in buffer"); 00508 mLastString=""; 00509 return mLastString; 00510 } 00511 // do my own reading 00512 int ll=Token::ReadCharacterData(mpStream,mLastString); 00513 if(ll<0) { 00514 std::stringstream errstr; 00515 errstr << "Text expected at " << FileLine(); 00516 throw Exception("TokenReader::TokenReader", errstr.str(), 50); 00517 } 00518 mLineCount+=ll; 00519 // prepend peek buffers string value (better: need rewind!) 00520 if(mPeekToken.IsString()) 00521 mLastString=mPeekToken.StringValue() + " " + mLastString; 00522 // invalidate buffer 00523 mPeekToken.SetNone(); 00524 return mLastString; 00525 } 00526 00527 // ReadSection() 00528 const std::string& TokenReader::ReadSection(void) { 00529 // record current level 00530 int clevel = Level(); 00531 // setup token writer for destination 00532 TokenWriter tw(TokenWriter::String); 00533 // token copy loop 00534 while(true) { 00535 // see whether we can grab and copy some character data 00536 std::string cdata=ReadCharacterData(); 00537 tw.WriteCharacterData(cdata); 00538 // break end of my level 00539 Token token; 00540 if(!Peek(token)) break; 00541 if(token.IsEnd() && !token.IsBegin() && Level()==clevel) 00542 break; 00543 // get and copy markup token 00544 Get(token); 00545 tw.Write(token); 00546 } 00547 // done 00548 mLastString=tw.Str(); 00549 return mLastString; 00550 } 00551 00552 00553 // Line() 00554 int TokenReader::Line(void) const { 00555 return mLineCount; 00556 } 00557 00558 // FileLine() 00559 std::string TokenReader::FileLine(void) const { 00560 if(mFileName!="") 00561 return "("+ mFileName + ":" + ToStringInteger(mLineCount) +")"; 00562 else 00563 return "(#" + ToStringInteger(mLineCount) +")"; 00564 } 00565 00566 } // namespace faudes libFAUDES 2.23h --- 2014.04.03 --- c++ api documentaion by doxygen |