Skip Menu |
 

This queue is for tickets about the YAML-LibYAML CPAN distribution.

Report information
The Basics
Id: 46172
Status: resolved
Priority: 0/
Queue: YAML-LibYAML

People
Owner: Nobody in particular
Requestors: JACKAL [...] cpan.org
Cc:
AdminCc:

Bug Information
Severity: Important
Broken in: 0.32
Fixed in: (no value)



Subject: Memory leaks at Load()
Download (untitled) / with headers
text/plain 226b
There is memory leaks at YAML::XS::Load() method. Tested on 2 platforms: FreeBSD 8.0-CURRENT #8: Sat Jan 17 19:42:04 MSK 2009 i386 FreeBSD 7.2-PRERELEASE #6: Mon Mar 23 23:09:10 MSK 2009 amd64 Attached test eats 1GB at i386
Subject: t_yaml_leak.pl
Download t_yaml_leak.pl
text/x-perl 482b
#!/usr/bin/perl use strict; use warnings; use YAML::XS; my $count = 1_000_000; my $yml = <<'TXT'; --- session_code: 649e116d890ad16306ef8d4491029db9 country: xx exchrate: 33.81 id: 14695373 message: xxxxx number: 1234 operator: xxx operator_full: xxx phone: 7701xxxxxxx plain: xxxxx price: 31.56037 TXT my $t = time(); foreach (1..$count) { my $y = YAML::XS::Load($yml); } $t = time() - $t; my $s = int($count/$t); print "YAML::XS::Load: time: $t sec; speed: $s req/sec\n";
From: dmaestro12
Download (untitled) / with headers
text/plain 715b
On Mon May 18 06:09:52 2009, JACKAL wrote: Show quoted text
> There is memory leaks at YAML::XS::Load() method. > > Tested on 2 platforms: > FreeBSD 8.0-CURRENT #8: Sat Jan 17 19:42:04 MSK 2009 i386 > FreeBSD 7.2-PRERELEASE #6: Mon Mar 23 23:09:10 MSK 2009 amd64 > > Attached test eats 1GB at i386
I ran into this problem also, and had to fix it for local use. Here are a couple of patches that "worked for me" Two problems had to be corrected: 1. Failure to release buffers allocated by yaml_parser_parse() 2. Reference count never went to zero on temporary hash key SV Tested only on linux Slackware 11.0.0 osname=linux, osvers=2.4.33.3, archname=i486-linux-thread-multi Perl 5.8.6 Should be pretty generic, though. DLS
Index: LibYAML/perl_libyaml.c =================================================================== --- LibYAML/perl_libyaml.c (revision 15429) +++ LibYAML/perl_libyaml.c (working copy) @@ -296,7 +296,7 @@ assert(SvPOK(key_node)); value_node = load_node(loader); hv_store_ent( - hash, key_node, value_node, 0 + hash, sv_2mortal(key_node), value_node, 0 ); }
Index: LibYAML/perl_libyaml.c =================================================================== --- LibYAML/perl_libyaml.c (revision 15428) +++ LibYAML/perl_libyaml.c (working copy) @@ -156,11 +156,15 @@ /* Keep calling load_node until end of stream */ while (1) { loader.document++; + /* We are through with the previous event - delete it! */ + yaml_event_delete(&loader.event); if (!yaml_parser_parse(&loader.parser, &loader.event)) goto load_error; if (loader.event.type == YAML_STREAM_END_EVENT) break; node = load_node(&loader); + /* We are through with the previous event - delete it! */ + yaml_event_delete(&loader.event); hv_clear(loader.anchors); if (! node) break; XPUSHs(sv_2mortal(node)); @@ -190,50 +194,80 @@ SV * load_node(perl_yaml_loader_t *loader) { + SV* return_sv = NULL; + /* This uses stack, but avoids (severe!) memory leaks */ + yaml_event_t uplevel_event; + + uplevel_event = loader->event; + /* Get the next parser event */ if (!yaml_parser_parse(&loader->parser, &loader->event)) goto load_error; + /* These events don't need yaml_event_delete */ + /* Some kind of error occurred */ + if (loader->event.type == YAML_NO_EVENT) + goto load_error; + /* Return NULL when we hit the end of a scope */ if (loader->event.type == YAML_DOCUMENT_END_EVENT || loader->event.type == YAML_MAPPING_END_EVENT || - loader->event.type == YAML_SEQUENCE_END_EVENT) return NULL; + loader->event.type == YAML_SEQUENCE_END_EVENT) { + /* restore the uplevel event, so it can be properly deleted */ + loader->event = uplevel_event; - /* Handle loading a mapping */ - if (loader->event.type == YAML_MAPPING_START_EVENT) { - SV *hash_ref; - char *tag = (char *)loader->event.data.mapping_start.tag; + return return_sv; + } + /* The rest all need cleanup */ + switch (loader->event.type) { + char *tag; + + /* Handle loading a mapping */ + case YAML_MAPPING_START_EVENT: + + tag = (char *)loader->event.data.mapping_start.tag; + /* Handle mapping tagged as a Perl hard reference */ - if (tag && strEQ(tag, TAG_PERL_REF)) - return load_scalar_ref(loader); - + if (tag && strEQ(tag, TAG_PERL_REF)) { + return_sv = load_scalar_ref(loader); + break; + } + /* Handle mapping tagged as a Perl typeglob */ - if (tag && strEQ(tag, TAG_PERL_GLOB)) - return load_glob(loader); + if (tag && strEQ(tag, TAG_PERL_GLOB)) { + return_sv = load_glob(loader); + break; + } + return_sv = load_mapping(loader, NULL); + break; - /* Load the mapping into a hash ref and return it */ - return load_mapping(loader, NULL); - } + /* Handle loading a sequence into an array */ + case YAML_SEQUENCE_START_EVENT: + return_sv = load_sequence(loader); + break; - /* Handle loading a sequence into an array */ - if (loader->event.type == YAML_SEQUENCE_START_EVENT) - return load_sequence(loader); + /* Handle loading a scalar */ + case YAML_SCALAR_EVENT: + return_sv = load_scalar(loader); + break; - /* Handle loading a scalar */ - if (loader->event.type == YAML_SCALAR_EVENT) - return load_scalar(loader); + /* Handle loading an alias node */ + case YAML_ALIAS_EVENT: + return_sv = load_alias(loader); + break; - /* Handle loading an alias node */ - if (loader->event.type == YAML_ALIAS_EVENT) - return load_alias(loader); + default: + croak(ERRMSG "Invalid event '%d' at top level", (int) loader->event.type); + } - /* Some kind of error occurred */ - if (loader->event.type == YAML_NO_EVENT) - croak(loader_error_msg(loader, NULL)); + yaml_event_delete(&loader->event); - croak(ERRMSG "Invalid event '%d' at top level", (int) loader->event.type); + /* restore the uplevel event, so it can be properly deleted */ + loader->event = uplevel_event; + return return_sv; + load_error: croak(loader_error_msg(loader, NULL)); }
Download (untitled) / with headers
text/plain 273b
I've also encountered leaks with YAML::XS 0.32 (not fixed in 0.34). Has anyone evaluated these patches? On Mon Jun 15 14:06:23 2009, http://dmaestro12.myopenid.com/ wrote: Show quoted text
> On Mon May 18 06:09:52 2009, JACKAL wrote:
> > There is memory leaks at YAML::XS::Load() method.
Download (untitled) / with headers
text/plain 542b
On Mon Feb 14 20:09:29 2011, ETHER wrote: Show quoted text
> I've also encountered leaks with YAML::XS 0.32 (not fixed in 0.34). Has > anyone evaluated these patches? > > On Mon Jun 15 14:06:23 2009, http://dmaestro12.myopenid.com/ wrote:
> > On Mon May 18 06:09:52 2009, JACKAL wrote:
> > > There is memory leaks at YAML::XS::Load() method.
I adapted the patch for the latest distro and it seems to work for me. Formerly, I was processing 200,000 YAML files and ending up with a footprint of 6 Gb of real memory. With this patch, it's less than 30 Mb.
Subject: 0001-Adjust-patch-for-RT-46172-to-cleanup-events.patch
From a1518056249a0ea204aa5f9d5bdb1f6a41b1351c Mon Sep 17 00:00:00 2001 From: brian d foy <brian.d.foy@gmail.com> Date: Tue, 29 Mar 2011 12:40:07 -0500 Subject: [PATCH 1/2] Adjust patch for RT 46172 to cleanup events https://rt.cpan.org/Ticket/Display.html?id=46172 --- LibYAML/perl_libyaml.c | 105 +++++++++++++++++++++++++++++++---------------- 1 files changed, 69 insertions(+), 36 deletions(-) diff --git a/LibYAML/perl_libyaml.c b/LibYAML/perl_libyaml.c index f79b571..3048fa7 100644 --- a/LibYAML/perl_libyaml.c +++ b/LibYAML/perl_libyaml.c @@ -159,11 +159,15 @@ Load(SV *yaml_sv) /* Keep calling load_node until end of stream */ while (1) { loader.document++; + /* We are through with the previous event - delete it! */ + yaml_event_delete(&loader.event); if (!yaml_parser_parse(&loader.parser, &loader.event)) goto load_error; if (loader.event.type == YAML_STREAM_END_EVENT) break; node = load_node(&loader); + /* We are through with the previous event - delete it! */ + yaml_event_delete(&loader.event); hv_clear(loader.anchors); if (! node) break; XPUSHs(sv_2mortal(node)); @@ -193,52 +197,81 @@ load_error: SV * load_node(perl_yaml_loader_t *loader) { + SV* return_sv = NULL; + /* This uses stack, but avoids (severe!) memory leaks */ + yaml_event_t uplevel_event; + + uplevel_event = loader->event; + /* Get the next parser event */ if (!yaml_parser_parse(&loader->parser, &loader->event)) goto load_error; + /* These events don't need yaml_event_delete */ + /* Some kind of error occurred */ + if (loader->event.type == YAML_NO_EVENT) + goto load_error; + /* Return NULL when we hit the end of a scope */ if (loader->event.type == YAML_DOCUMENT_END_EVENT || loader->event.type == YAML_MAPPING_END_EVENT || - loader->event.type == YAML_SEQUENCE_END_EVENT) return NULL; + loader->event.type == YAML_SEQUENCE_END_EVENT) { + /* restore the uplevel event, so it can be properly deleted */ + loader->event = uplevel_event; + return return_sv; + } - /* Handle loading a mapping */ - if (loader->event.type == YAML_MAPPING_START_EVENT) { - SV *hash_ref; - char *tag = (char *)loader->event.data.mapping_start.tag; + /* The rest all need cleanup */ + switch (loader->event.type) { + char *tag; - /* Handle mapping tagged as a Perl hard reference */ - if (tag && strEQ(tag, TAG_PERL_REF)) - return load_scalar_ref(loader); - - /* Handle mapping tagged as a Perl typeglob */ - if (tag && strEQ(tag, TAG_PERL_GLOB)) - return load_glob(loader); + /* Handle loading a mapping */ + case YAML_MAPPING_START_EVENT: + tag = (char *)loader->event.data.mapping_start.tag; - /* Load the mapping into a hash ref and return it */ - return load_mapping(loader, NULL); - } + /* Handle mapping tagged as a Perl hard reference */ + if (tag && strEQ(tag, TAG_PERL_REF)) { + return_sv = load_scalar_ref(loader); + break; + } - /* Handle loading a sequence into an array */ - if (loader->event.type == YAML_SEQUENCE_START_EVENT) - return load_sequence(loader); + /* Handle mapping tagged as a Perl typeglob */ + if (tag && strEQ(tag, TAG_PERL_GLOB)) { + return_sv = load_glob(loader); + break; + } - /* Handle loading a scalar */ - if (loader->event.type == YAML_SCALAR_EVENT) - return load_scalar(loader); + return_sv = load_mapping(loader, NULL); + break; - /* Handle loading an alias node */ - if (loader->event.type == YAML_ALIAS_EVENT) - return load_alias(loader); + /* Handle loading a sequence into an array */ + case YAML_SEQUENCE_START_EVENT: + return_sv = load_sequence(loader); + break; - /* Some kind of error occurred */ - if (loader->event.type == YAML_NO_EVENT) - croak(loader_error_msg(loader, NULL)); + /* Handle loading a scalar */ + case YAML_SCALAR_EVENT: + return_sv = load_scalar(loader); + break; - croak(ERRMSG "Invalid event '%d' at top level", (int) loader->event.type); + /* Handle loading an alias node */ + case YAML_ALIAS_EVENT: + return_sv = load_alias(loader); + break; -load_error: - croak(loader_error_msg(loader, NULL)); + default: + croak(ERRMSG "Invalid event '%d' at top level", (int) loader->event.type); + } + + yaml_event_delete(&loader->event); + + /* restore the uplevel event, so it can be properly deleted */ + loader->event = uplevel_event; + + return return_sv; + + load_error: + croak(loader_error_msg(loader, NULL)); } /* @@ -265,7 +298,7 @@ load_mapping(perl_yaml_loader_t *loader, char *tag) assert(SvPOK(key_node)); value_node = load_node(loader); hv_store_ent( - hash, key_node, value_node, 0 + hash, sv_2mortal(key_node), value_node, 0 ); } @@ -345,7 +378,7 @@ load_scalar(perl_yaml_loader_t *loader) class = tag + strlen(prefix); scalar = sv_setref_pvn(newSV(0), class, string, strlen(string)); SvUTF8_on(scalar); - return scalar; + return scalar; } if (loader->event.data.scalar.style == YAML_PLAIN_SCALAR_STYLE) { @@ -858,11 +891,11 @@ dump_scalar(perl_yaml_dumper_t *dumper, SV *node, yaml_char_t *tag) style = YAML_SINGLE_QUOTED_SCALAR_STYLE; } if (!SvUTF8(node)) { - /* copy to new SV and promote to utf8 */ - SV *utf8sv = sv_mortalcopy(node); + /* copy to new SV and promote to utf8 */ + SV *utf8sv = sv_mortalcopy(node); - /* get string and length out of utf8 */ - string = SvPVutf8(utf8sv, string_len); + /* get string and length out of utf8 */ + string = SvPVutf8(utf8sv, string_len); } } yaml_scalar_event_initialize( -- 1.6.5.4
applied bdfoy's patch in 0.35. thanks to all who contributed!


This service is sponsored and maintained by Best Practical Solutions and runs on Perl.org infrastructure.

Please report any issues with rt.cpan.org to rt-cpan-admin@bestpractical.com.