GNU Octave  3.8.0
A high-level interpreted language, primarily intended for numerical computations, mostly compatible with Matlab
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
lo-regexp.cc
Go to the documentation of this file.
1 /*
2 
3 Copyright (C) 2012 John W. Eaton
4 Copyright (C) 2005-2013 David Bateman
5 Copyright (C) 2002-2005 Paul Kienzle
6 
7 This file is part of Octave.
8 
9 Octave is free software; you can redistribute it and/or modify it
10 under the terms of the GNU General Public License as published by the
11 Free Software Foundation; either version 3 of the License, or (at your
12 option) any later version.
13 
14 Octave is distributed in the hope that it will be useful, but WITHOUT
15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 for more details.
18 
19 You should have received a copy of the GNU General Public License
20 along with Octave; see the file COPYING. If not, see
21 <http://www.gnu.org/licenses/>.
22 
23 */
24 
25 #ifdef HAVE_CONFIG_H
26 #include <config.h>
27 #endif
28 
29 #include <list>
30 #include <sstream>
31 #include <string>
32 #include <vector>
33 
34 #if defined (HAVE_PCRE_H)
35 #include <pcre.h>
36 #elif defined (HAVE_PCRE_PCRE_H)
37 #include <pcre/pcre.h>
38 #endif
39 
40 #include "Matrix.h"
41 #include "base-list.h"
42 #include "lo-error.h"
43 #include "oct-locbuf.h"
44 #include "quit.h"
45 #include "lo-regexp.h"
46 #include "str-vec.h"
47 
48 // Define the maximum number of retries for a pattern
49 // that possibly results in an infinite recursion.
50 #define PCRE_MATCHLIMIT_MAX 10
51 
52 // FIXME: should this be configurable?
53 #define MAXLOOKBEHIND 10
54 
55 static bool lookbehind_warned = false;
56 
57 // FIXME: don't bother collecting and composing return values
58 // the user doesn't want.
59 
60 void
62 {
63  if (data)
64  pcre_free (static_cast<pcre *> (data));
65 }
66 
67 void
69 {
70  // If we had a previously compiled pattern, release it.
71  free ();
72 
73  size_t max_length = MAXLOOKBEHIND;
74 
75  size_t pos = 0;
76  size_t new_pos;
77  int inames = 0;
78  std::ostringstream buf;
79 
80  while ((new_pos = pattern.find ("(?", pos)) != std::string::npos)
81  {
82  if (pattern.at (new_pos + 2) == '<'
83  && !(pattern.at (new_pos + 3) == '='
84  || pattern.at (new_pos + 3) == '!'))
85  {
86  // The syntax of named tokens in pcre is "(?P<name>...)" while
87  // we need a syntax "(?<name>...)", so fix that here. Also an
88  // expression like
89  // "(?<first>\w+)\s+(?<last>\w+)|(?<last>\w+),\s+(?<first>\w+)"
90  // should be perfectly legal, while pcre does not allow the same
91  // named token name on both sides of the alternative. Also fix
92  // that here by replacing name tokens by dummy names, and dealing
93  // with the dummy names later.
94 
95  size_t tmp_pos = pattern.find_first_of ('>', new_pos);
96 
97  if (tmp_pos == std::string::npos)
98  {
99  (*current_liboctave_error_handler)
100  ("regexp: syntax error in pattern");
101  return;
102  }
103 
104  std::string tmp_name =
105  pattern.substr (new_pos+3, tmp_pos-new_pos-3);
106 
107  bool found = false;
108 
109  for (int i = 0; i < nnames; i++)
110  {
111  if (named_pats(i) == tmp_name)
112  {
113  named_idx.resize (dim_vector (inames+1, 1));
114  named_idx(inames) = i;
115  found = true;
116  break;
117  }
118  }
119 
120  if (! found)
121  {
122  named_idx.resize (dim_vector (inames+1, 1));
123  named_idx(inames) = nnames;
124  named_pats.append (tmp_name);
125  nnames++;
126  }
127 
128  if (new_pos - pos > 0)
129  buf << pattern.substr (pos, new_pos-pos);
130  if (inames < 10)
131  buf << "(?P<n00" << inames++;
132  else if (inames < 100)
133  buf << "(?P<n0" << inames++;
134  else
135  buf << "(?P<n" << inames++;
136 
137  pos = tmp_pos;
138  }
139  else if (pattern.at (new_pos + 2) == '<')
140  {
141  // Find lookbehind operators of arbitrary length (ie like
142  // "(?<=[a-z]*)") and replace with a maximum length operator
143  // as PCRE can not yet handle arbitrary length lookahead
144  // operators. Use the string length as the maximum length to
145  // avoid issues.
146 
147  int brackets = 1;
148  size_t tmp_pos1 = new_pos + 2;
149  size_t tmp_pos2 = tmp_pos1;
150 
151  while (tmp_pos1 < pattern.length () && brackets > 0)
152  {
153  char ch = pattern.at (tmp_pos1);
154 
155  if (ch == '(')
156  brackets++;
157  else if (ch == ')')
158  {
159  if (brackets > 1)
160  tmp_pos2 = tmp_pos1;
161 
162  brackets--;
163  }
164 
165  tmp_pos1++;
166  }
167 
168  if (brackets != 0)
169  {
170  buf << pattern.substr (pos, new_pos - pos) << "(?";
171  pos = new_pos + 2;
172  }
173  else
174  {
175  size_t tmp_pos3 = pattern.find_first_of ("*+", tmp_pos2);
176 
177  if (tmp_pos3 != std::string::npos && tmp_pos3 < tmp_pos1)
178  {
179  if (!lookbehind_warned)
180  {
181  lookbehind_warned = true;
182  (*current_liboctave_warning_handler)
183  ("%s: arbitrary length lookbehind patterns are only supported up to length %d",
184  who.c_str (), MAXLOOKBEHIND);
185  }
186 
187  buf << pattern.substr (pos, new_pos - pos) << "(";
188 
189  size_t i;
190 
191  if (pattern.at (tmp_pos3) == '*')
192  i = 0;
193  else
194  i = 1;
195 
196  for (; i < max_length + 1; i++)
197  {
198  buf << pattern.substr (new_pos, tmp_pos3 - new_pos)
199  << "{" << i << "}";
200  buf << pattern.substr (tmp_pos3 + 1,
201  tmp_pos1 - tmp_pos3 - 1);
202  if (i != max_length)
203  buf << "|";
204  }
205  buf << ")";
206  }
207  else
208  buf << pattern.substr (pos, tmp_pos1 - pos);
209 
210  pos = tmp_pos1;
211  }
212  }
213  else
214  {
215  buf << pattern.substr (pos, new_pos - pos) << "(?";
216  pos = new_pos + 2;
217  }
218 
219  }
220 
221  buf << pattern.substr (pos);
222 
223  const char *err;
224  int erroffset;
225  std::string buf_str = buf.str ();
226 
227  int pcre_options
228  = ((options.case_insensitive () ? PCRE_CASELESS : 0)
229  | (options.dotexceptnewline () ? 0 : PCRE_DOTALL)
230  | (options.lineanchors () ? PCRE_MULTILINE : 0)
231  | (options.freespacing () ? PCRE_EXTENDED : 0));
232 
233  data = pcre_compile (buf_str.c_str (), pcre_options, &err, &erroffset, 0);
234 
235  if (! data)
236  (*current_liboctave_error_handler)
237  ("%s: %s at position %d of expression", who.c_str (),
238  err, erroffset);
239 }
240 
242 regexp::match (const std::string& buffer)
243 {
244  regexp::match_data retval;
245 
246  std::list<regexp::match_element> lst;
247 
248  int subpatterns;
249  int namecount;
250  int nameentrysize;
251  char *nametable;
252  size_t idx = 0;
253 
254  pcre *re = static_cast <pcre *> (data);
255 
256  pcre_fullinfo (re, 0, PCRE_INFO_CAPTURECOUNT, &subpatterns);
257  pcre_fullinfo (re, 0, PCRE_INFO_NAMECOUNT, &namecount);
258  pcre_fullinfo (re, 0, PCRE_INFO_NAMEENTRYSIZE, &nameentrysize);
259  pcre_fullinfo (re, 0, PCRE_INFO_NAMETABLE, &nametable);
260 
261  OCTAVE_LOCAL_BUFFER (int, ovector, (subpatterns+1)*3);
262  OCTAVE_LOCAL_BUFFER (int, nidx, namecount);
263 
264  for (int i = 0; i < namecount; i++)
265  {
266  // Index of subpattern in first two bytes MSB first of name.
267  // Extract index.
268  nidx[i] = (static_cast<int> (nametable[i*nameentrysize])) << 8
269  | static_cast<int> (nametable[i*nameentrysize+1]);
270  }
271 
272  while (true)
273  {
274  OCTAVE_QUIT;
275 
276  int matches = pcre_exec (re, 0, buffer.c_str (),
277  buffer.length (), idx,
278  (idx ? PCRE_NOTBOL : 0),
279  ovector, (subpatterns+1)*3);
280 
281  if (matches == PCRE_ERROR_MATCHLIMIT)
282  {
283  // Try harder; start with default value for MATCH_LIMIT
284  // and increase it.
285  (*current_liboctave_warning_handler)
286  ("your pattern caused PCRE to hit its MATCH_LIMIT; trying harder now, but this will be slow");
287 
288  pcre_extra pe;
289 
290  pcre_config (PCRE_CONFIG_MATCH_LIMIT,
291  static_cast <void *> (&pe.match_limit));
292 
293  pe.flags = PCRE_EXTRA_MATCH_LIMIT;
294 
295  int i = 0;
296  while (matches == PCRE_ERROR_MATCHLIMIT
297  && i++ < PCRE_MATCHLIMIT_MAX)
298  {
299  OCTAVE_QUIT;
300 
301  pe.match_limit *= 10;
302  matches = pcre_exec (re, &pe, buffer.c_str (),
303  buffer.length (), idx,
304  (idx ? PCRE_NOTBOL : 0),
305  ovector, (subpatterns+1)*3);
306  }
307  }
308 
309  if (matches < 0 && matches != PCRE_ERROR_NOMATCH)
310  {
311  (*current_liboctave_error_handler)
312  ("%s: internal error calling pcre_exec; error code from pcre_exec is %i",
313  who.c_str (), matches);
314  return retval;
315  }
316  else if (matches == PCRE_ERROR_NOMATCH)
317  break;
318  else if (ovector[1] <= ovector[0] && ! options.emptymatch ())
319  {
320  // Zero length match. Skip to next char.
321  idx = ovector[0] + 1;
322  if (idx < buffer.length ())
323  continue;
324  else
325  break;
326  }
327  else
328  {
329  int pos_match = 0;
330  Matrix token_extents (matches-1, 2);
331 
332  for (int i = 1; i < matches; i++)
333  {
334  if (ovector[2*i] >= 0 && ovector[2*i+1] > 0
335  && (i == 1 || ovector[2*i] != ovector[2*i-2]
336  || ovector[2*i-1] != ovector[2*i+1]))
337  {
338  token_extents(pos_match,0) = double (ovector[2*i]+1);
339  token_extents(pos_match++,1) = double (ovector[2*i+1]);
340  }
341  }
342 
343  token_extents.resize (pos_match, 2);
344 
345  double start = double (ovector[0]+1);
346  double end = double (ovector[1]);
347 
348  const char **listptr;
349  int status = pcre_get_substring_list (buffer.c_str (), ovector,
350  matches, &listptr);
351 
352  if (status == PCRE_ERROR_NOMEMORY)
353  {
354  (*current_liboctave_error_handler)
355  ("%s: cannot allocate memory in pcre_get_substring_list",
356  who.c_str ());
357  return retval;
358  }
359 
360  string_vector tokens (pos_match);
361  string_vector named_tokens (nnames);
362  int pos_offset = 0;
363  pos_match = 0;
364 
365  for (int i = 1; i < matches; i++)
366  {
367  if (ovector[2*i] >= 0 && ovector[2*i+1] > 0)
368  {
369  if (i == 1 || ovector[2*i] != ovector[2*i-2]
370  || ovector[2*i-1] != ovector[2*i+1])
371  {
372  if (namecount > 0)
373  {
374  // FIXME: Should probably do this with a map()
375  // rather than a linear search. However,
376  // the number of captured, named expressions
377  // is usually pretty small (< 4)
378  for (int j = 0; j < namecount; j++)
379  {
380  if (nidx[j] == i)
381  {
382  named_tokens(named_idx(j)) =
383  std::string (*(listptr+i-pos_offset));
384  break;
385  }
386  }
387  }
388 
389  tokens(pos_match++) = std::string (*(listptr+i));
390  }
391  else
392  pos_offset++;
393  }
394  }
395 
396  std::string match_string = std::string (*listptr);
397 
398  pcre_free_substring_list (listptr);
399 
400  regexp::match_element new_elem (named_tokens, tokens, match_string,
401  token_extents, start, end);
402  lst.push_back (new_elem);
403 
404  if (ovector[1] <= ovector[0])
405  {
406  // Zero length match. Skip to next char.
407  idx = ovector[0] + 1;
408  if (idx <= buffer.length ())
409  continue;
410  }
411  else
412  idx = ovector[1];
413 
414  if (options.once () || idx >= buffer.length ())
415  break;
416  }
417  }
418 
419  retval = regexp::match_data (lst, named_pats);
420 
421  return retval;
422 }
423 
424 bool
425 regexp::is_match (const std::string& buffer)
426 {
427  regexp::match_data rx_lst = match (buffer);
428 
429  return rx_lst.size () > 0;
430 }
431 
434 {
435  octave_idx_type len = buffer.length ();
436 
437  Array<bool> retval (dim_vector (len, 1));
438 
439  for (octave_idx_type i = 0; i < buffer.length (); i++)
440  retval(i) = is_match (buffer(i));
441 
442  return retval;
443 }
444 
445 // Declare rep_token_t used in processing replacement string
446 typedef struct
447 {
448  size_t pos;
449  int num;
450 } rep_token_t;
451 
452 
453 std::string
454 regexp::replace (const std::string& buffer, const std::string& replacement)
455 {
456  std::string retval;
457 
458  regexp::match_data rx_lst = match (buffer);
459 
460  size_t num_matches = rx_lst.size ();
461 
462  if (num_matches == 0)
463  {
464  retval = buffer;
465  return retval;
466  }
467 
468  // Identify replacement tokens; build a vector of group numbers in
469  // the replacement string so that we can quickly calculate the size
470  // of the replacement.
471 
472  // FIXME: All code assumes that only 10 tokens ($0-$9) exist.
473  // $11 represents $1 followed by the character '1' rather than
474  // the eleventh capture buffer.
475 
476  std::string repstr = replacement;
477  std::vector<rep_token_t> tokens;
478  tokens.reserve (5); // Reserve memory for 5 pattern replacements
479 
480  for (size_t i=0; i < repstr.size (); i++)
481  {
482  if (repstr[i] == '\\')
483  {
484  if (i < repstr.size () - 1 && repstr[i+1] == '$')
485  {
486  repstr.erase (i,1); // erase backslash
487  i++; // skip over '$'
488  continue;
489  }
490  if (i < repstr.size () - 1 && repstr[i+1] == '\\')
491  {
492  repstr.erase (i,1); // erase 1st backslash
493  continue;
494  }
495  }
496  else if (repstr[i] == '$')
497  {
498  if (i < repstr.size () - 1 && isdigit (repstr[i+1]))
499  {
500  rep_token_t tmp_token;
501 
502  tmp_token.pos = i;
503  tmp_token.num = repstr[i+1]-'0';
504  tokens.push_back (tmp_token);
505  }
506  }
507  }
508 
509  std::string rep;
510  int num_tokens = tokens.size ();
511 
512  if (num_tokens > 0)
513  {
514  // Determine replacement length
515  const size_t replen = repstr.size () - 2*num_tokens;
516  int delta = 0;
518  for (size_t i = 0; i < num_matches; i++)
519  {
520  OCTAVE_QUIT;
521 
522  double start = p->start ();
523  double end = p->end ();
524 
525  const Matrix pairs (p->token_extents ());
526  size_t pairlen = 0;
527  for (int j = 0; j < num_tokens; j++)
528  {
529  if (tokens[j].num == 0)
530  pairlen += static_cast<size_t> (end - start) + 1;
531  else if (tokens[j].num <= pairs.rows ())
532  pairlen += static_cast<size_t> (pairs(tokens[j].num-1,1)
533  - pairs(tokens[j].num-1,0)) + 1;
534  }
535  delta += (static_cast<int> (replen + pairlen)
536  - static_cast<int> (end - start + 1));
537  p++;
538  }
539 
540  // Build replacement string
541  rep.reserve (buffer.size () + delta);
542  size_t from = 0;
543  p = rx_lst.begin ();
544  for (size_t i = 0; i < num_matches; i++)
545  {
546  OCTAVE_QUIT;
547 
548  double start = p->start ();
549  double end = p->end ();
550 
551  const Matrix pairs (p->token_extents ());
552  rep.append (&buffer[from], static_cast<size_t> (start - 1) - from);
553  from = static_cast<size_t> (end);
554 
555  size_t cur_pos = 0;
556 
557  for (int j = 0; j < num_tokens; j++)
558  {
559  rep.append (&repstr[cur_pos], (tokens[j].pos) - cur_pos);
560  cur_pos = tokens[j].pos+2;
561 
562  int k = tokens[j].num;
563  if (k == 0)
564  {
565  // replace with entire match
566  rep.append (&buffer[static_cast<size_t> (end - 1)],
567  static_cast<size_t> (end - start) + 1);
568  }
569  else if (k <= pairs.rows ())
570  {
571  // replace with group capture
572  rep.append (&buffer[static_cast<size_t> (pairs(k-1,0)-1)],
573  static_cast<size_t> (pairs(k-1,1)
574  - pairs(k-1,0)) + 1);
575  }
576  else
577  {
578  // replace with nothing
579  }
580  }
581  if (cur_pos < repstr.size ())
582  rep.append (&repstr[cur_pos], repstr.size () - cur_pos);
583 
584  p++;
585  }
586  rep.append (&buffer[from], buffer.size () - from);
587  }
588  else
589  {
590  // Determine repstr length
591  const size_t replen = repstr.size ();
592  int delta = 0;
594  for (size_t i = 0; i < num_matches; i++)
595  {
596  OCTAVE_QUIT;
597  delta += static_cast<int> (replen)
598  - static_cast<int> (p->end () - p->start () + 1);
599  p++;
600  }
601 
602  // Build replacement string
603  rep.reserve (buffer.size () + delta);
604  size_t from = 0;
605  p = rx_lst.begin ();
606  for (size_t i = 0; i < num_matches; i++)
607  {
608  OCTAVE_QUIT;
609  rep.append (&buffer[from],
610  static_cast<size_t> (p->start () - 1) - from);
611  from = static_cast<size_t> (p->end ());
612  rep.append (repstr);
613  p++;
614  }
615  rep.append (&buffer[from], buffer.size () - from);
616  }
617 
618  retval = rep;
619  return retval;
620 }