package ReDIF::Parser;

##  Copyright (c) 1997-2001 Ivan Kurmanov. All rights reserved.
##
##  This code is free software; you can redistribute it and/or modify it
##  under the same terms as Perl itself.

=head1 NAME

ReDIF::Parser - interface to the ReDIF parsing and validation software
(ReDIF-perl)

=head1 SYNOPSIS

    ###  import some functions
    use ReDIF::Parser qw( &redif_open_file 
			  &redif_parse_string
			  &redif_get_next_template 
			  &redif_get_next_template_good_or_bad
			  &redif_open_dir 
			  &redif_open_dir_recursive
			  );

    ###  prepare the parser
    ReDIF::Parser::set_parser_options( message_threshold => 2,
				       build_good_template_text => 1 );

    ###  open a file
    my $file = "t/datafile.rdf";
    redif_open_file ( $file );

    ###  alternative: open all ReDIF files in a directory
    my $dir = "~/my-ReDIF/";
    redif_open_dir ( $dir );
    redif_open_dir ( $dir, 1 );  ## read the files in alphabetical order

    ###  alternative: open all ReDIF files in a directory and subdirs
    my $dir = "~/RePEc/remo/";
    redif_open_dir_recursive ( $dir );

    ###  alternative:  parse a string variable
    redif_parse_string( $redif_string );


    ###  get a template parsed
    my $template = redif_get_next_template( ); 

    ###  extract template data
    my $t_type   = $template -> {'template-type'} [0]; # note the quotes
    my $title    = $template -> {title} [0]; 
    my $author_1 = $template -> {author} [0] {name} [0]; 
    my $author_2 = $template -> {author} [1] {name} [0]; 
    my $author_3 = $template -> {author} [2] {name} [0]; 

    my $authors = join ( ', ', $author_1, $author_2, $author_3 );
    print "Paper '$title' by $authors\n";

    ###  alternative: retreive the next template
    my $template = redif_get_next_template_good_or_bad( ); 

    ###  check template's validity
    if( $template->{RESULT} eq 'bad' ) {
	print "template is invalid and has following problems:\n";
        print $template->{PROBLEMS};
    }

    if( $template->{RESULT} eq 'good' ) {
	print "template is valid and may be printed out in canonical form as:\n";
        print $template->{TEXT};
    }


=head1 DESCRIPTION 

Use this module if you need to process ReDIF data.  

ReDIF stands for Research Document Information Format, and you
can find it's official description at http://repec.org.

Basically it is a simple textual metadata format, based on the
"attribute: value" syntax.  It is used for the RePEc project.  The
main record types of ReDIF are academia-related.  In particular,
primary purpose of ReDIF was to describe economics discipline.

The ReDIF parser itself is implemented in a set of modules below
ReDIF::Parser::.  So this module is just an interface.  Using
ReDIF::Parser is very simple, but there is a number of things in this
manpage that you should read carefully before using it.

The main parser module (ReDIF::Parser::Core, which you should not use
directly) does all kinds of job for you, and returns the
parsing/validation result back to you on a template-by-template basis.

A ReDIF file may contain zero or more ReDIF templates.  If one
template is invalid, other templates still can be OK -- the templates
are independent between each other.

The work the parser does for you depends on the options you set.  I
call them "parser options".  B<The parser options won't effect the
validation in any way, but they may effect the way parser returns the
processing results back to you>.

The module has a very simple procedural interface and is limited to
processing one file at a time only.  You can not have several
independent instances of the Parser in a single perl interpreter.

=head1 FUNCTIONS

Each of the described functions has two names, first name starts with
'redif_', the other just says what the function is about.  The only
difference is that 'redif_'-named functions can be imported to your
package namespaces: they are included into the ReDIF::Parser's
@EXPORT_OK list.  After that you can call them directly in your code
as "redif_open_file(...);"

Functions without "redif_" cannot be (legally) imported by you, but
you can call them through package-qualified names, e.g.
"ReDIF::Parser::open_file(...)".

=over 4

=item redif_set_parser_options ( OPT1, VAL1 [, OPT2, VAL2 ...] )

=item set_parser_options ( OPT1, VAL1 [, OPT2, VAL2 ...] )


Sets the given parser option(s) to the corresponding given value(s).
Note that previously set parser options are not cleared, unless you
provide a new value for it.  So you can use this function
incrementally - you can call it several times in a row for one option
at a time.  It will have the same result as if you did it in one call
for several options.

The recognized parser options are described below in section PARSER
OPTIONS.  You should note that ReDIF::Parser sets some options by
default.

Also note that perl (often) interpretes '=E<gt>' combination as a comma
sign ','.


=item redif_open_file ( FILENAME [, POSITION ] )

=item open_file ( FILENAME [, POSITION ] )

The parser will prepare to process the ReDIF file FILENAME and will
open it for reading.  If positive POSITION is given, the file will be
read starting from that POSITION.  (So you may process the file
starting from a specific position.)

Returns true on success, false otherwise.



=item redif_open_dir ( DIRNAME [, SORTFLAG ] )

=item open_dir ( DIRNAME [, SORTFLAG ] )

The parser will prepare to process every file in the directory
DIRNAME, whose name ends with '.rdf' (case-insensitive), as ReDIF.
get_next_template (and company) functions will seamlessly move on to
the next file whenever the previous file is over.  You can always find
out the actual file being read by checking the FILENAME element of the
template hash structure.

Only readable normal files (not-softlinks) are used and everything
else is ignored.

If SORTFLAG is true, the list of all files found is alphabetically
sorted before starting its processing.

For reading all files in a directory and it's subdirectories, see
redif_open_dir_recursive() function.

Returns number of files found, that will be attempted for reading and
zero if no appropriate files found or in case of an error.


=item redif_open_dir_recursive ( DIRNAME [, SORTFLAG ] )

=item open_dir_recursive ( DIRNAME [, SORTFLAG ] )

Differs from redif_open_dir() and open_dir() functions in that it
searches for ReDIF files in all subdirectories of the DIRNAME.

Returns number of files found, that will be attempted for reading and
zero if no appropriate files found or in case of an error.


=item redif_parse_string ( STRING ) 

=item parse_string ( STRING ) 

Parse the STRING containing ReDIF template(s).  In many ways it works
like open_file().  Following ...get_next_template...() calls would
return the actual data.

Returns 1.


=item redif_get_next_template ( ) 

=item get_next_template ( )

The parser will read the next (first) template in the previously
opened file.  

While reading the data from the ReDIF file, the parser will validate
it against a so-called 'ReDIF Specification File' also known as
B<redif.spec>.  This file contains a machine-readable specification of
ReDIF rules and restrictions.  This file is supposed to reflect the
official ReDIF specification document.  A mistake in a ReDIF template
may generate a parsing error or warning.  If an error found in a
template, the template is considered invalid and should not be used in
applications.  A warning doesn't lead to such consequence, but it
usually leads to a partial loss of the template's data in an
application: the attribute which caused the warning or part of it's
value will be omitted from the parsing result -- i.e. will not reach
the application.

Invalid templates are ignored and the parser will continue processing
the file until it finds a valid one or an end of file reached.

All that is done to make applications' life easier.  They are
guaranteed to recieve only valid ReDIF templates with only valid
values of the attributes.

The function returns a reference to template hash structure on
success, or false on the End Of File.  The template hash structure is
described elsewhere in this document.

If a successful redif_open_dir() or redif_open_dir_recursive() call
preceded then redif_get_next_template() upon reaching an EOF would
seamlessly open the next file, until the end of the file-list prepared
by any of the open_dir.. functions.


=item redif_get_next_template_good_or_bad ( ) 

=item get_next_template_good_or_bad ( )

That function is almost like the previous one: get_next_template().
The difference is that this function also returns errorneous
templates.

So before actually using any template data got from this function, you
should check template hash element RESULT for being equal to string 'good'.

=back


=head1 PARSER OPTIONS

These are the options recognized by the parser.  Most options' role is
to turn ON or OFF some of the features of the parser.

=over 4

=item remove_newline_from_values

When positive, the parser will normalize whitespace in attribute values, by a
substitution operator $value =~ s/\s+/ /g;

=item build_template_hash

When set to true, the parser will make a hash representation of each
parsed template.  Each template attribute, converted to lower case,
will become a hash key.  The attribute value will be put into an
array.  The array reference will become the hash value.  So the
stucture of the final hash will look like:

  { 
      attribute         => [ value ],
      another-attribute => [ value1, value2 ],
      ...
  }

Note: you will normally get a reference to the hash.  See the TEMPLATE
HASH STRUCTURE section for more information.

This option is ON by default and you usually will need it.


=item build_good_template_text

This option, when enabled, makes parser generate a clean textual
representation of your template, as good-old B<rere> script does, and
put it into the TEXT value of the resulting template hash structure.

As attributes are read from the file and their values are checked, all
good processed attribute-value pairs are appended to a variable as
"$attribute: $value\n".  The $attribute here is a lower-case attribute
name, the $value is the atribute's value after some (required)
processing.  At least the following is done to the value: 

 - start and end whitespace stripped;

 - linebreaks removed.

Further processing may be done by the attribute type, as specified in
redif.spec ('ReDIF Specification').

If an attribute has an empty value, it is not included.  The "X-"
attributes are included if the 'x_attributes' parsing option is
enabled.

If the option is on and the underlying template is valid, then the
TEXT value of the template hash structure must actually be a valid
ReDIF template.  If that is not the case, it means we have a serious
bug -- so please let us know.

This options is OFF by default.


=item x_attributes

If set to true, the parser will process and return to the application
all encountered attributes, starting with "X-".  Otherwise, they will
be ignored.  One might be interested to know, that "X-" attributes all
may appear inside clusters, e.g. 'author-x-hair: blonde' will be
returned as "x-hair" attribute insie the author's cluster structure
(in case build_template_hash option is on).

Default value is false.


=item message_threshold 

This option is useful only if the application needs to process not the
template itself, but the warnings and error messages generated by the
parser, while processing a template.  

The parser during the validation process generates several kinds of
"messages".  Message types range from (debugging) notice (0) to (rare)
critical error (4), with very common warning (2) and error (3) in
between.  When the template structure returned to the application by
one of the I<get_next_template> functions, $template-E<gt>{PROBLEMS}
contains a string with the messages, which where generated by the
template ("\n"-separated).  The message_threshold option sets the
minimal status of messages that should be included and not ignored.

If it is set to 2, warnings, errors and critical errors will be
included.  If it is set to 3, only errors and critical errors will be
included.  Message threshold set to 0 (zero) is useful for debugging
purposes only.

This also effects the REPORT value in the resulting template hash.

Default value for this option is 2.


=item quote_source

This option is used by B<rech> script for ReDIF data checking and 
usual application won't need it.

When set to true, it makes parser collect source data lines (prefixing
them with "E<gt> ") before each message (error or warning).  The resulting
nice readable and self-explanatory template-checking report is stored
in the REPORT value of the template hash.

OFF by default.


=item utf8_output

Makes parser to pass all resulting values, messages and so on in the
UTF-8 encoding.

By default parser will return data in iso-8859-1 (latin-1), which is a
default ReDIF encoding.

Note, however, that if the option is false, the application will be
prevented from processing templates, whose contents cannot be
expressed in iso-8859-1 encoding (for instance, a template containing
a single cyrillic letter).

Also please note, that all internal processing in the parser is done
in UTF-8 encoding and if you request output in latin-1, this places an
additional burden of decoding from UTF-8 to latin-1 and that may
influence performance slightly.

See doc\README.unicode for more information.

By default this option is OFF.



=item HashT 

HashT is a deprecated legacy compatibility feature.  Previous versions
of ReDIF-perl always stored resulting template hash structure in the
global variable %::HashT.

If true, the template hash structure reference, returned by the
get_next_template... functions would point to %::HashT.  Otherwise, no
variable in the main package namespace will ever be touched.






=back


=head1 TEMPLATE HASH STRUCTURE

The structure of the parsing result value, returned by
...get_next_template... functions.  The I<get_next_template ()> and
I<get_next_template_good_or_bad()> and their redif_... counterparts
return a hash reference.  The following items are present in that hash
as keys.

=over 4 

=item FILENAME

The name of the file being parsed.

=item STARTFPOS

The byte-position within the file, where the template begins.

=item RESULT

Template validity: 'good' or 'bad'.  When using I<get_next_template()>
or I<redif_get_next_template()> it is always 'good' (because invalid
templates are skipped).

=item ENCODING

The encoding the template data is in.  Three values are possible:
normal: 'utf-8' and 'latin1'; special: 'invalid'.  The ENCODING is
directly related to the 'utf8_output' parser option.  If utf8_output
is set to false, but the template data can't be expressed in
iso-8859-1 encoding, then ENCODING will have 'invalid' value.

Otherwise, it will be set according to the 'utf8_output' parser
option.

=item WARNINGS

The number of warnings generated by the template parsing.  Independent
of the 'message_threshold' parser option.

=item ERRORS

The number of errors generated by the template parsing.  Independent
of the 'message_threshold' parser option.

=item MESSAGES

The number of messages (e.g. warnings, errors) generated by the
template parsing that have status equal to or greater then
'message_threshold' parser option.

See also 'message_threshold' parser option description above.

=item PROBLEMS

A string containing new-line-separated messages generated by the
template parsing that have status equal to or greater than the
'message_threshold' parser option.

=item REPORT

A string containing the quoted source data (prefixed with "E<gt> ") and
the messages generated by the template parsing that have status equal
to or greater than the 'message_threshold' parser option.

Would look like B<rech> output. 

If parser option 'quote_source' is set to false, then REPORT is equal
to PROBLEMS value.

=item TEXT

if parser option 'build_good_template_text' is ON, the whole template
is in one string in that value.  For more info, read about
'build_good_template_text' parser option above.  You may think of this
as a 'canonical' template form.

=item MD5SUM

If during ReDIF-perl installation you had Digest::MD5 perl module
installed and you chose to enable MD5 checksum calculation for
templates, then in MD5SUM item you should see base64-representation of
the 128-bit checksum, calculated on the template source lines with the
MD5 digest algorithm.  The sum it this (base64) representation will be
a 24 characters long string.  It can be used to track template
mutations in time.

If you in a script need to know, if the feature is enabled or not, you
can check ReDIF::Parser::calculate_md5_checksum constant, e.g.:

   if ( ReDIF::Parser::calculate_md5_checksum ) { ... } 

=back

B<ATTENTION!> That is not all.  If the parser option
'build_template_hash' is ON, the template hash contains the template
data itself.  Lower-case items (keys) of the template hash represent
the attributes of the original template.  Some of them represent whole
clusters, which in turn have their own attributes.

Attribute values are items of the lists, referenced by the attribute
key.  

Let's look at a usual situation as an example.  Assume we process a
'ReDIF-Paper 1.0' template.  If $template variable contains template
hash structure, and the template had an attribute of 'Title', then
$template-E<gt>{title} would return an array reference and
$template-E<gt>{title}[0] would return the value of the attribute 'Title'
in the original ReDIF template. ($template-E<gt>{title}[1] could return
value of the second attribute 'Title' in the original template, if
there was a second attribute with such name, but currently none of
ReDIF template types allows attribute 'title' to be repeated.)

If we go on with an example, let's assume that 'author-name' attribute
was present in the template with some non-empty value.  Then
$template-E<gt>{author}[0] would return a hash reference, containing the
members of the PERSON cluster under the attribute of 'author'.
$template-E<gt>{author}[0]{name}[0] would represent the first
'author-name' attribute and would return it's value.


=head1 BUGS & TO DO

Well, a user should be able to directly specify what redif.spec file
to use.

=head1 COPYRIGHT & LICENSE

Copyright (c) 1997-2001 Ivan Kurmanov. All rights reserved.

This code is free software; you can redistribute it and/or modify it
under the same terms as Perl itself.

=head1 AUTHOR

Ivan Kurmanov, for the RePEc project

=cut

##### '


BEGIN { 

$VERSION = "2.30";
#$VERSION = do { my @r=(q$Revision: 1.39 $=~/\d+/g); sprintf "%d."."%02d"x$#r,@r }; 
#                       $Id: Parser.pm,v 1.39 2003/01/04 14:31:02 ivan Exp $
}


use strict;

use Exporter;

use vars qw( @EXPORT_OK @ISA  %Options $fileopen $directory_mode @files_list );

@ISA = ( 'Exporter' );
@EXPORT_OK = qw( &redif_open_file 
                 &redif_get_next_template
                 &redif_get_next_template_good_or_bad
                 &redif_set_parser_options 

                 &redif_set_parser_option 
                 &redif_get_parser_option 

		 &redif_parse_string

		 &redif_open_dir
		 &redif_open_dir_recursive
                   );


use ReDIF::init;

ReDIF::initialize( {'silent_init' => 1 } );

use ReDIF::Spec ();

my $spec_file = $ReDIF::CONFIG{spec_full_name} ;

my $spec_o = ReDIF::Spec -> new( $spec_file );

use ReDIF::setup_conf;
use ReDIF::Parser::Input;
use ReDIF::Parser::Core;

&ReDIF::Parser::Core::multiple_values_bug_fix();


%Options = (
	    'HashT' => 0,
	    'quote_source' => 0,
#	    'build_template_hash' => 1,
	    'x_attributes' => 0,
	    'rech_mode' => 0,
	    'message_threshold' => 2,
	    'use_parser_input' => 1,
	    'redif_specification' => $spec_o,
	    'remove_newline_from_values' => 0,
);


ReDIF::Parser::Core::init( \%Options );


#######################################################################################
###    FUNCTIONS   ####################################################################
#######################################################################################

###   set_parser_options

sub set_parser_options { 
    return redif_set_parser_options ( @_ ) ;
}

sub redif_set_parser_options {
    my %opts = @_;
    foreach my $k ( keys %opts ) {
	my $v = $opts{$k};
	$Options{$k} = $v;
    }

    &ReDIF::Parser::Core::init( \%Options ) ;
} 



###  open_file

sub open_file { 
    return redif_open_file ( @_ ) ;
}

sub redif_open_file {
    my ($file, $pos, $showname) = @_;
    
    if ($pos) {
	if ( not ReDIF::Parser::Input::start_file ( $file, $pos ) ) {
	    return 0;
	}
    } else {
	if ( not ReDIF::Parser::Input::start_file ( $file, $pos ) ) {
	    return 0;
	}
    }
    if ( defined $showname ) {$file = $showname;}
    
    $fileopen = $file;
    &ReDIF::Parser::Core::starting_new_file( $file );
    
    return 1;
}



###  parse string 

sub parse_string {
    return redif_parse_string( @_ );
}

sub redif_parse_string {
    my $string = shift;
    
    my $filename = "string buffer";
    $fileopen = $filename;

    &ReDIF::Parser::Input::start_string( $string );

    &ReDIF::Parser::Core::starting_new_file( $filename );

    return 1;
}



###   get_next_template

sub get_next_template {
    return redif_get_next_template( @_ );
}


sub redif_get_next_template {

   return undef if not $fileopen;
   
   my @data;
   my $R = 1;

   while( $R ) {
       $R = &ReDIF::Parser::Input::extract_next_attribute();

       if ( not $R ) {
	   undef $fileopen;
	   last;
       }

       if( ReDIF::Parser::Core::get_current_template_status() eq 'good' ) { 
	   return ReDIF::Parser::Core::get_current_template();  
       }
   }
   
   if ( $directory_mode and scalar @files_list ) {
       while( 1 ) {    ###  open next file from the list, until successful
	   last if
	       redif_open_file ( shift @files_list );

	   if( not scalar @files_list ) {   ### quit if no more files
	       $directory_mode = 0;
	       return 0;
	   }
       }
       return redif_get_next_template ();
   }
   
   return 0;
}



###   get_next_template_good_or_bad

sub get_next_template_good_or_bad {
    return redif_get_next_template_good_or_bad( @_ );
}

sub redif_get_next_template_good_or_bad {

   return undef if not $fileopen;
   
   my @data;
   my $R = 1;

   while( $R ) {

       $R = &ReDIF::Parser::Input::extract_next_attribute();

       if ( not $R ) {
	   undef $fileopen;
	   last;
       }

       if( ReDIF::Parser::Core::get_current_template_status() !~ m/unfin$/ ) { 
	   return ReDIF::Parser::Core::get_current_template();  
       }
   }
   
   if ( $directory_mode and scalar @files_list ) {
       while( 1 ) {    ###  open next file from the list, until successful
	   last if
	       redif_open_file ( shift @files_list );

	   if( not scalar @files_list ) {   ### quit if no more files
	       $directory_mode = 0;
	       return 0;
	   }
       }
       return redif_get_next_template_good_or_bad ();
   }
   
   return 0;
}


###   open dir

sub open_dir { 
    return redif_open_dir ( @_ ) ;
}

sub redif_open_dir {
    my $dir = shift;
    my $sort_flag = shift;

    @files_list = ();
    
    opendir ( DIR , $dir )
	or die "can't open dir $dir" ;
    if ($dir !~ /[\\\/]$/) { $dir .= "\/"; }
    
    my ( $c, $f, $a );
    $c = 0;
    while ( defined ( $f = readdir DIR ) )
    {
#	print "dir elem: $f\n";
	$a++;
	next unless $f =~ /\.rdf$/i;     # filtering out non-ReDIF files
	my $fname = "$dir$f";            # making a full name
	next unless -f $fname && -e _ && -r _ ;   # a check
	push @files_list, $fname;        # saving a name in a list
	$c ++;                        # control counter
    }

    if ( $sort_flag ) { @files_list = sort @files_list; }
    closedir DIR;

    if ( not $c ) {
	$directory_mode = 0;
	return 0;

    } else {
	$directory_mode = 1;

	while( 1 ) {    ###  open next file from the list, until successful
	    last if
		redif_open_file ( shift @files_list );
	    
	    if( not scalar @files_list ) {   ### quit if no more files
		$directory_mode = 0;
		return 0;
	    }
	}

	return $c;
    }
}




###   open dir recursive

use File::Find;

sub add_element_to_files_list {

    return if $_ !~ /\.rdf$/i;     # filtering out non-ReDIF files

    return unless -f $_ && -e _ && -r _;   # a redability check

    push @files_list, $File::Find::name;   # saving a name in a list
}


sub open_dir_recursive { 
    return redif_open_dir_recursive ( @_ ) ;
}

sub redif_open_dir_recursive {
    my $dir = shift;
    my $sort_flag = shift;

    @files_list = ();
    
    find( \&add_element_to_files_list,  $dir );
	  
    if ( $sort_flag ) { 
	@files_list = sort @files_list; 
    }
    
    my $c = scalar @files_list;

    if ( not $c ) {
	$directory_mode = 0;
	return 0;

    } else {
	$directory_mode = 1;

	while( 1 ) {    ###  open next file from the list, until successful
	    last if
		redif_open_file ( shift @files_list );
	    
	    if( not scalar @files_list ) {   ### quit if no more files
		$directory_mode = 0;
		return 0;
	    }
	}

	return $c;
    }
}



########################################################################################
####   That's all !   ##################################################################


1;
