My Favorite Tokenizer

I have used this tokenizer countless times. It is easy to translate. I coded something similar in 32 bit Microsoft assemplber. If one learns a new language, this is a great function to translate. If one has to tokenize something, this is great code to cut copy paste, and use as a starting point.

void tokenize( string& input_string, string& seperators, vector& tokens)
{
  size_t token_offset=0, seperator_offset, token_len;

  for (seperator_offset=0; seperator_offset!=string::npos && token_offset != string::npos; ) {
    token_offset = input_string.find_first_not_of( seperators, seperator_offset);
    if (token_offset != string::npos) {
      seperator_offset = input_string.find_first_of( seperators, token_offset);
      token_len = (seperator_offset==string::npos)? (input_string.size() - token_offset):(seperator_offset - token_offset);
      string token( input_string.substr( token_offset, token_len));
      tokens.push_back( token);
    }
  }
}