memory exhausted

R

Ren Py

filename1 = "e:\\jxkh\\tools\\getKhbz.rb"
file1 = open(filename1,'w')
file1.print "def getKhbz(did)\n"
file1.print " case did\n"

cs = (1..2498).to_a
for j in cs do
str = " when '" + j.to_s + "';return '" + j.to_s + "';\n"
file1.print str
end

file1.print " else\n"
file1.print " return ''\n"
file1.print " end\n"
file1.print " end\n"

E:\jxkh\tools>ruby getKhbz.rb
getKhbz.rb:2499: memory exhausted
when '2497';return '2497';

"memory exhausted"!!!! why? please
 
R

Ryan Davis

E:\jxkh\tools>ruby getKhbz.rb
getKhbz.rb:2499: memory exhausted
when '2497';return '2497';

"memory exhausted"!!!! why? please

soooo many things open to critique, but not with regard to memory.
 
L

Luis Parravicini

I don't know why you use a 2498 lines case only to return the same
value you matched in the when statement. Anyway, I've tested the
script with ruby 1.8.6 and I got the same error (memory exhausted). If
I remove two lines of the case, it runs fine.

grepping the ruby source shows regex.c is the only file with that
string. And thats how far I got. Any ideas?
 
M

MenTaLguY

grepping the ruby source shows regex.c is the only file with that
string. And thats how far I got. Any ideas?

The error is coming from the parser. Here's a simpler test case using integers:

eval <<EOS
case x
#{ (0..2498).map { |n| "when #{n}: nil\n" }.join('') }
end
EOS

We know that it's happening at parse-time rather than run-time, because the out-of-memory error occurs before Ruby has a chance to complain that 'x' is undefined.

In short, the Ruby parser apparently cannot cope with case statements which have more than 2498 branches (at least with the build of Ruby I've got here, changing 2498 to 2497 is enough to avoid the memory issue).

-mental
 
T

Tim Pease

The error is coming from the parser. Here's a simpler test case using integers:

eval <<EOS
case x
#{ (0..2498).map { |n| "when #{n}: nil\n" }.join('') }
end
EOS

We know that it's happening at parse-time rather than run-time, because the out-of-memory error occurs before Ruby has a chance to complain that 'x' is undefined.

In short, the Ruby parser apparently cannot cope with case statements which have more than 2498 branches (at least with the build of Ruby I've got here, changing 2498 to 2497 is enough to avoid the memory issue).

Probably hitting a lex/yacc semantic stack limit.

I wonder if you could do the same kind of thing with a
begin/rescue/ensure/end clause?

Hmmm ....

eval <<EOS
begin
raise
#{ (0..1999).map { "rescue;\n" }.join('') }
end
EOS

(eval):2003: compile error (SyntaxError)
(eval):2003: memory exhausted


Sure enough! But it only took 1999 rescue clauses. 1998 does not
cause it to happen.

Blessings,
TwP
 
N

Nobuyoshi Nakada

Hi,

At Wed, 11 Apr 2007 04:56:23 +0900,
Tim Pease wrote in [ruby-talk:247418]:
Probably hitting a lex/yacc semantic stack limit.

Yes, it will work by increasing YYMAXDEPTH.
I wonder if you could do the same kind of thing with a
begin/rescue/ensure/end clause?

Hmmm ....

eval <<EOS
begin
raise
#{ (0..1999).map { "rescue;\n" }.join('') }
end
EOS

(eval):2003: compile error (SyntaxError)
(eval):2003: memory exhausted

A patch to inverse the order of shift/reduce.


Index: parse.y
===================================================================
--- parse.y (revision 12169)
+++ parse.y (working copy)
@@ -273,5 +273,5 @@ static void top_local_setup();
%type <node> bodystmt compstmt stmts stmt expr arg primary command command_call method_call
%type <node> expr_value arg_value primary_value
-%type <node> if_tail opt_else case_body cases opt_rescue exc_list exc_var opt_ensure
+%type <node> if_tail opt_else case_body cases opt_rescue rescue exc_list exc_var opt_ensure
%type <node> args when_args call_args call_args2 open_args paren_args opt_paren_args
%type <node> command_args aref_args opt_block_arg block_arg var_ref var_lhs
@@ -376,5 +376,7 @@ bodystmt : compstmt
$$ = $1;
if ($2) {
- $$ = NEW_RESCUE($1, $2, $3);
+ $$ = $2;
+ $$->nd_head = $1;
+ $$->nd_else = $3;
}
else if ($3) {
@@ -1599,14 +1601,16 @@ primary : literal
}
}
- | kCASE expr_value opt_terms
- case_body
- kEND
+ | kCASE expr_value opt_terms cases opt_else kEND
{
- $$ = NEW_CASE($2, $4);
+ $4->nd_head->nd_next = $5;
+ $4->nd_head = $2;
+ $$ = $4;
fixpos($$, $2);
}
- | kCASE opt_terms case_body kEND
+ | kCASE opt_terms cases opt_else kEND
{
- $$ = $3;
+ $3->nd_head->nd_next = $4;
+ $$ = $3->nd_body;
+ rb_gc_force_recycle((VALUE)$3);
}
| kCASE opt_terms kELSE compstmt kEND
@@ -1871,7 +1875,6 @@ brace_block : '{'
case_body : kWHEN when_args then
compstmt
- cases
{
- $$ = NEW_WHEN($2, $4, $5);
+ $$ = NEW_WHEN($2, $4, 0);
}
;
@@ -1887,11 +1890,34 @@ when_args : args
;

-cases : opt_else
- | case_body
+cases : case_body
+ {
+ $$ = NEW_CASE($1, $1);
+ }
+ | cases
+ case_body
+ {
+ $$ = $1;
+ $$->nd_head->nd_next = $2;
+ $$->nd_head = $2;
+ }
+ ;
+
+opt_rescue : none
+ | opt_rescue
+ rescue
+ {
+ if ($1) {
+ $1->nd_head->nd_head = $$;
+ $1->nd_head = $$;
+ $$ = $1;
+ }
+ else {
+ $$ = NEW_RESCUE($2, $2, 0);
+ }
+ }
;

-opt_rescue : kRESCUE exc_list exc_var then
+rescue : kRESCUE exc_list exc_var then
compstmt
- opt_rescue
{
if ($3) {
@@ -1899,8 +1925,7 @@ opt_rescue : kRESCUE exc_list exc_var th
$5 = block_append($3, $5);
}
- $$ = NEW_RESBODY($2, $5, $6);
+ $$ = NEW_RESBODY($2, $5, 0);
fixpos($$, $2?$2:$5);
}
- | none
;
 
N

Nobuyoshi Nakada

Hi,

At Thu, 12 Apr 2007 04:19:02 +0900,
Nobuyoshi Nakada wrote in [ruby-talk:247527]:
A patch to inverse the order of shift/reduce.

And for if/elsif/else.


Index: parse.y
===================================================================
--- parse.y (revision 12169)
+++ parse.y (working copy)
@@ -273,5 +273,5 @@ static void top_local_setup();
%type <node> bodystmt compstmt stmts stmt expr arg primary command command_call method_call
%type <node> expr_value arg_value primary_value
-%type <node> if_tail opt_else case_body cases opt_rescue exc_list exc_var opt_ensure
+%type <node> if_then opt_else case_body cases opt_rescue exc_list exc_var opt_ensure
%type <node> args when_args call_args call_args2 open_args paren_args opt_paren_args
%type <node> command_args aref_args opt_block_arg block_arg var_ref var_lhs
@@ -1553,11 +1553,9 @@ primary : literal
fixpos($$, $1);
}
- | kIF expr_value then
- compstmt
- if_tail
- kEND
+ | if_then opt_else kEND
{
- $$ = NEW_IF(cond($2), $4, $5);
- fixpos($$, $2);
+ $1->nd_else->nd_else = $2;
+ $$ = $1->nd_body;
+ rb_gc_force_recycle((VALUE)$1);
if (cond_negative(&$$->nd_cond)) {
NODE *tmp = $$->nd_body;
@@ -1744,18 +1742,26 @@ do : term
;

-if_tail : opt_else
- | kELSIF expr_value then
- compstmt
- if_tail
+opt_else : none
+ | kELSE compstmt
{
- $$ = NEW_IF(cond($2), $4, $5);
- fixpos($$, $2);
+ $$ = $2;
}
;

-opt_else : none
- | kELSE compstmt
+if_then : kIF expr_value then
+ compstmt
{
- $$ = $2;
+ $$ = NEW_IF(cond($2), $4, 0);
+ fixpos($$, $2);
+ $$ = NEW_IF(0, $$, $$);
+ }
+ | if_then kELSIF expr_value then
+ compstmt
+ {
+ NODE *elsif = NEW_IF(cond($3), $5, 0);
+ fixpos(elsif, $3);
+ $$ = $1;
+ $$->nd_else->nd_else = elsif;
+ $$->nd_else = elsif;
}
;
 
R

Robert Klemme

At Thu, 12 Apr 2007 04:19:02 +0900,
Nobuyoshi Nakada wrote in [ruby-talk:247527]:
A patch to inverse the order of shift/reduce.

And for if/elsif/else.

But what do we gain by this? I haven't seen this error in years of Ruby
(neither in my own code nor in postings here) which leads me to believe
that it's actually not an issue in practice. The test case seemed
extremely artificial and I guess CASE statements of that size are quite
a lot more inefficient than Hash lookups or other strategies.

Kind regards

robert
 
R

Robert Klemme

Eh, a little bit of correctness, I guess. It's probably best if the parser can handle anything which is syntactically valid Ruby, even if it's a bit dumb.

I think it's worth the effort only if a) it really accepts "unlimited"
cases and b) it doesn't cost performance. (Haven't checked either)
(an alternative would be to actually have an explicit limit on the number of 'when' or 'rescue' branches in the grammar)

Yep.

robert
 
N

Nobuyoshi Nakada

Hi,

At Fri, 13 Apr 2007 15:50:11 +0900,
Robert Klemme wrote in [ruby-talk:247769]:
I think it's worth the effort only if a) it really accepts "unlimited"
cases and b) it doesn't cost performance. (Haven't checked either)

In this case, yes for a, and almost yes for b, I guess.

Since there was a typo in [ruby-talk:247527], merged patch of
[ruby-talk:247527] and [ruby-talk:247595].


Index: parse.y
===================================================================
--- parse.y (revision 12203)
+++ parse.y (working copy)
@@ -273,5 +273,5 @@ static void top_local_setup();
%type <node> bodystmt compstmt stmts stmt expr arg primary command command_call method_call
%type <node> expr_value arg_value primary_value
-%type <node> if_tail opt_else case_body cases opt_rescue exc_list exc_var opt_ensure
+%type <node> if_then opt_else case_body cases opt_rescue rescue exc_list exc_var opt_ensure
%type <node> args when_args call_args call_args2 open_args paren_args opt_paren_args
%type <node> command_args aref_args opt_block_arg block_arg var_ref var_lhs
@@ -376,5 +376,7 @@ bodystmt : compstmt
$$ = $1;
if ($2) {
- $$ = NEW_RESCUE($1, $2, $3);
+ $$ = $2;
+ $$->nd_head = $1;
+ $$->nd_else = $3;
}
else if ($3) {
@@ -1553,11 +1555,9 @@ primary : literal
fixpos($$, $1);
}
- | kIF expr_value then
- compstmt
- if_tail
- kEND
+ | if_then opt_else kEND
{
- $$ = NEW_IF(cond($2), $4, $5);
- fixpos($$, $2);
+ $1->nd_else->nd_else = $2;
+ $$ = $1->nd_body;
+ rb_gc_force_recycle((VALUE)$1);
if (cond_negative(&$$->nd_cond)) {
NODE *tmp = $$->nd_body;
@@ -1599,14 +1599,16 @@ primary : literal
}
}
- | kCASE expr_value opt_terms
- case_body
- kEND
+ | kCASE expr_value opt_terms cases opt_else kEND
{
- $$ = NEW_CASE($2, $4);
+ $4->nd_head->nd_next = $5;
+ $4->nd_head = $2;
+ $$ = $4;
fixpos($$, $2);
}
- | kCASE opt_terms case_body kEND
+ | kCASE opt_terms cases opt_else kEND
{
- $$ = $3;
+ $3->nd_head->nd_next = $4;
+ $$ = $3->nd_body;
+ rb_gc_force_recycle((VALUE)$3);
}
| kCASE opt_terms kELSE compstmt kEND
@@ -1744,18 +1746,26 @@ do : term
;

-if_tail : opt_else
- | kELSIF expr_value then
- compstmt
- if_tail
+opt_else : none
+ | kELSE compstmt
{
- $$ = NEW_IF(cond($2), $4, $5);
- fixpos($$, $2);
+ $$ = $2;
}
;

-opt_else : none
- | kELSE compstmt
+if_then : kIF expr_value then
+ compstmt
{
- $$ = $2;
+ $$ = NEW_IF(cond($2), $4, 0);
+ fixpos($$, $2);
+ $$ = NEW_IF(0, $$, $$);
+ }
+ | if_then kELSIF expr_value then
+ compstmt
+ {
+ NODE *elsif = NEW_IF(cond($3), $5, 0);
+ fixpos(elsif, $3);
+ $$ = $1;
+ $$->nd_else->nd_else = elsif;
+ $$->nd_else = elsif;
}
;
@@ -1871,7 +1881,6 @@ brace_block : '{'
case_body : kWHEN when_args then
compstmt
- cases
{
- $$ = NEW_WHEN($2, $4, $5);
+ $$ = NEW_WHEN($2, $4, 0);
}
;
@@ -1887,11 +1896,34 @@ when_args : args
;

-cases : opt_else
- | case_body
+cases : case_body
+ {
+ $$ = NEW_CASE($1, $1);
+ }
+ | cases
+ case_body
+ {
+ $$ = $1;
+ $$->nd_head->nd_next = $2;
+ $$->nd_head = $2;
+ }
;

-opt_rescue : kRESCUE exc_list exc_var then
+opt_rescue : none
+ | opt_rescue
+ rescue
+ {
+ if ($1) {
+ $1->nd_head->nd_head = $2;
+ $1->nd_head = $2;
+ $$ = $1;
+ }
+ else {
+ $$ = NEW_RESCUE($2, $2, 0);
+ }
+ }
+ ;
+
+rescue : kRESCUE exc_list exc_var then
compstmt
- opt_rescue
{
if ($3) {
@@ -1899,8 +1931,7 @@ opt_rescue : kRESCUE exc_list exc_var th
$5 = block_append($3, $5);
}
- $$ = NEW_RESBODY($2, $5, $6);
+ $$ = NEW_RESBODY($2, $5, 0);
fixpos($$, $2?$2:$5);
}
- | none
;

@@ -6109,4 +6140,5 @@ rb_id2name(id)
{
char *name;
+ st_data_t data;

if (id < tLAST_TOKEN) {
@@ -6119,6 +6151,6 @@ rb_id2name(id)
}

- if (st_lookup(sym_rev_tbl, id, (st_data_t *)&name))
- return name;
+ if (st_lookup(sym_rev_tbl, id, &data))
+ return (char *)data;

if (is_attrset_id(id)) {
@@ -6333,5 +6365,5 @@ rb_parser_free(ptr)
NODE **prev = &parser_heap, *n;

- while (n = *prev) {
+ while ((n = *prev) != 0) {
if (n->u1.node == ptr) {
*prev = n->u2.node;
 

Ask a Question

Want to reply to this thread or ask your own question?

You'll need to choose a username for the site, which only take a couple of moments. After that, you can post your question and our members will help you out.

Ask a Question

Members online

Forum statistics

Threads
473,744
Messages
2,569,484
Members
44,903
Latest member
orderPeak8CBDGummies

Latest Threads

Top