perl - Converting code using hash to array

Question

I am charged with making a Perl module more efficient. It takes log entries which are currently uniquely identified by date and server name and inserts them into our mysql database for later processing.

I've posted the original file and my attempt below. My code doesn't insert into the database. I'm sure its a simple array handling problem but not sure.

Original file

package UlsaSpectrumAnalyser;

use strict;
use warnings;

use Data::Dumper;
use EnmServiceGroup;
use StatsDB;
use DBI;
use StatsTime;

sub new
{
    my $klass = shift;
    my $self = bless {}, $klass;
    return $self;
}

sub init($$$$)
{
    my ($self,$r_cliArgs,$r_incr,$dbh) = @_;
    $self->{'site'} = $r_cliArgs->{'site'};
    $self->{'siteId'} = $r_cliArgs->{'siteId'};
    $self->{'date'} = $r_cliArgs->{'date'};
    if ( exists $r_incr->{'UlsaSpectrumAnalyser'} )
    {
        $self->{'r_UlsaSpectrumAnalyser'} = $r_incr->{'UlsaSpectrumAnalyser'}->{'r_UlsaSpectrumAnalyser'};
    }
    else
    {
        $self->{'r_UlsaSpectrumAnalyser'} = {};
    }

    my @subscriptions = ();
    $self->{'serverMap'} = {};
    foreach my $service( "pmservice", "saservice" ) {
        my $r_serverMap = enmGetServiceGroupInstances($self->{'site'}, $self->{'date'},$service);
        while ( my ($server,$serverId) = each %{$r_serverMap} ) {
            push ( @subscriptions, {'server' => $server, 'prog' => 'JBOSS'} );
            $self->{'serverMap'}->{$server} = $serverId;
        }
    }
    return @subscriptions;
}

sub handle($$$$$$$)
{
    my ($self,$timestamp,$host,$program,$severity,$message,$messageSize) = @_;
    if ( $::DEBUG > 9 ) { print "UlsaSpectrumAnalyser::handle got message from $host $program : $message
"; }

    # Skip any warnings/errors
    if ( $severity ne 'info' ) {
        return;
    }

    my ($time)=$timestamp=~/(d{4}-d{2}-d{2}Td{2}:d{2}:d{2}).*/;
    my ($epochtime) = getDateTimeInMilliSeconds($timestamp);
    if ( $::DEBUG > 3 ) { print "UlsaSpectrumAnalyser::handle got message from $time $host $program : $message
"; }

    #Sample Log
    #2017-09-08 14:55:36,028 INFO [com.X.Y.itpf.EVENT_LOGGER] (ajp-executor-threads - 18) [administrator, Spectrum Analyzer, DETAILED, #ULSA_COMPONENT_FFT, MeContext=lienb4003, Samples=65510; FileParsingTime(ms)=50; FastFourierTime(ms)=370; PostProcessingTime(ms)=3; #ChartScalingTime(ms)=1; TotalTime(ms)=424]

    if( $message =~ /.*ULSA_COMPONENT_FFT,s+(S+),s+Samples=(d+);s+FileParsingTimeS+=(d+);s+FastFourierTimeS+=(d+);s+PostProcessingTimeS+=(d+);s+ChartScalingTimeS+=(d+);s+TotalTimeS+=(d+)]/ ) {

       my $activity = $epochtime . '@@' . $host;  #TODO Handle array instead of hash. Working on separately
       my $serverid = $self->{'serverMap'}->{$host};
       $self->{'r_UlsaSpectrumAnalyser'}->{$activity} = {
               'time'                  => $time,
               'epochtime'             => $epochtime,
               'serverid'              => $serverid,
               'source'                => $1,
               'sample'                => $2,
               'file_parsing_time'     => $3,
               'fast_fourier_time'     => $4,
               'post_processing_time'  => $5,
               'chart_scaling_time'    => $6,
               'total_time'            => $7};
    }

}

sub handleExceeded($$$)
{
    my ($self, $host, $program) = @_;
}

sub done($$$)
{
    my ($self,$dbh,$r_incr) = @_;
    my $tmpDir = '/data/tmp';
    my $date=$self->{'date'};
    if (exists $ENV{'TMP_DIR'})
    {
        $tmpDir = $ENV{'TMP_DIR'};
    }
    my $bcpFileUlsaAnalyserLogs = "$tmpDir/ulsa_spectrum_analyser_logs.bcp";
    open (BCP, "> $bcpFileUlsaAnalyserLogs") or die "Failed to open $bcpFileUlsaAnalyserLogs";

    foreach my $activity (sort keys %{$self->{'r_UlsaSpectrumAnalyser'}}) {

       print BCP $self->{'siteId'} . "	" .
           $self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'serverid'} . "	" .
           $self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'time'} . "	" .
           $self->{'r_UlsaSpiUectrumAnalyser'}->{$activity}->{'epochtime'} . "	" .
           $self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'source'} . "	" .
           $self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'sample'} . "	" .
           $self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'file_parsing_time'} . "	" .
           $self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'fast_fourier_time'} . "	" .
           $self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'post_processing_time'} . "	" .
           $self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'chart_scaling_time'} . "	" .
           $self->{'r_UlsaSpectrumAnalyser'}->{$activity}->{'total_time'} . "
";
    }
    close BCP;
    dbDo( $dbh, "DELETE FROM enm_ulsa_spectrum_analyser_logs  WHERE siteid = $self->{'siteId'} AND time BETWEEN '$date 00:00:00' AND '$date 23:59:59'" )
    or die "Failed to delete from enm_ulsa_spectrum_analyser_logs" . $dbh->errstr;

    dbDo( $dbh, "LOAD DATA INFILE '$bcpFileUlsaAnalyserLogs' INTO TABLE enm_ulsa_spectrum_analyser_logs" )
    or die "Failed to load new data from '$bcpFileUlsaAnalyserLogs' file to 'enm_ulsa_spectrum_analyser_logs' table" . $dbh->errstr;
    unlink($bcpFileUlsaAnalyserLogs);

    $r_incr->{'UlsaSpectrumAnalyser'} = {
                                 'r_UlsaSpectrumAnalyser' => $self->{'r_UlsaSpectrumAnalyser'}
                                 };
}

1;

My attempt

package UlsaSpectrumAnalyser;

use strict;
use warnings;

use Data::Dumper;
use StatsDB;
use DBI;
use StatsTime;

sub new
{
    my $klass = shift;
    my $self = bless {}, $klass;
    return $self;
}

sub init($$$$)
{
    my ($self,$r_cliArgs,$r_incr,$dbh) = @_;
    $self->{'site'} = $r_cliArgs->{'site'};
    $self->{'siteId'} = $r_cliArgs->{'siteId'};
    $self->{'date'} = $r_cliArgs->{'date'};
    $self->{'r_instrDataEvent'}->{'ulsaStats'} = [];


    my @subscriptions = ();
    $self->{'serverMap'} = {};
    foreach my $service( "pmservice", "saservice" ) {
        my $r_serverMap = enmGetServiceGroupInstances($self->{'site'}, $self->{'date'},$service);
        while ( my ($server,$serverId) = each %{$r_serverMap} ) {
            push ( @subscriptions, {'server' => $server, 'prog' => 'JBOSS'} );
            $self->{'serverMap'}->{$server} = $serverId;
        }
    }
    return @subscriptions;
}

sub handle($$$$$$$)
{
    my ($self,$timestamp,$host,$program,$severity,$message,$messageSize) = @_;
    if ( $::DEBUG > 9 ) { print "UlsaSpectrumAnalyser::handle got message from $host $program : $message
"; }

    # Skip any warnings/errors
    if ( $severity ne 'info' ) {
        return;
    }

    my ($time)=$timestamp=~/(d{4}-d{2}-d{2}Td{2}:d{2}:d{2}).*/;
    my ($epochtime) = getDateTimeInMilliSeconds($timestamp);
    if ( $::DEBUG > 3 ) { print "UlsaSpectrumAnalyser::handle got message from $time $host $program : $message
"; }

    #Sample Log
    #2017-09-08 14:55:36,028 INFO [com.X.y.itpf.EVENT_LOGGER] (ajp-executor-threads - 18) [administrator, Spectrum Analyzer, DETAILED, #ULSA_COMPONENT_FFT, MeContext=lienb4003, Samples=65510; FileParsingTime(ms)=50; FastFourierTime(ms)=370; PostProcessingTime(ms)=3; #ChartScalingTime(ms)=1; TotalTime(ms)=424]

    if( $message =~ /.*ULSA_COMPONENT_FFT,s+(S+),s+Samples=(d+);s+FileParsingTimeS+=(d+);s+FastFourierTimeS+=(d+);s+PostProcessingTimeS+=(d+);s+ChartScalingTimeS+=(d+);s+TotalTimeS+=(d+)]/ ) {
        my $serverid = $self->{'r_instrDataEvent'}->{'serverid'};
        my %event = (
            'time'                 => $time,
            'epochtime'            => $epochtime,
            'serverid'             => $serverid,
            'source'               => $1,
            'sample'               => $2,
            'file_parsing_time'    => $3,
            'fast_fourier_time'    => $4,
            'post_processing_time' => $5,
            'chart_scaling_time'   => $6,
            'total_time'           => $7
        );
        push @{$self->{'r_instrDataEvent'}->{'ulsaStats'}}, \%event;
    }
}

sub handleExceeded($$$)
{
    my ($self, $host, $program) = @_;
}

sub done($$$)
{
    my ($self,$dbh,$r_incr) = @_;

    my $bcpFileUlsaAnalyserLogs = getBcpFilename("enm_ulsa_spectrum_analyser_logs ");
    open (BCP, "> $bcpFileUlsaAnalyserLogs") or die "Failed to open $bcpFileUlsaAnalyserLogs";

    foreach my $activity (@{$self->{'r_instrDataEvent'}->{'ulsaStats'}}) {

       print BCP $self->{'siteId'} . "	" .
           $activity->{'serverid'} . "	" .
           $activity->{'time'} . "	" .
           $activity->{'epochtime'} . "	" .
           $activity->{'source'} . "	" .
           $activity->{'sample'} . "	" .
           $activity->{'file_parsing_time'} . "	" .
           $activity->{'fast_fourier_time'} . "	" .
           $activity->{'post_processing_time'} . "	" .
           $activity->{'chart_scaling_time'} . "	" .
           $activity->{'total_time'} . "
";
    }
    close BCP;

    dbDo( $dbh, sprintf("DELETE FROM enm_ulsa_spectrum_analyser_logs  WHERE siteid = %d AND time BETWEEN '%s' AND '%s'",
    $self->{'siteId'}, $self->{'r_instrDataEvent'}->{'ulsaStats'}->[0]->{'epochtime'},$self->{'r_instrDataEvent'}->{'ulsaStats'}->[$#{$self->{'r_instrDataEvent'}->{'ulsaStats'}}]->{'epochtime'}))
    or die "Failed to delete from enm_ulsa_spectrum_analyser_logs" . $dbh->errstr;#'

    dbDo( $dbh, "LOAD DATA INFILE '$bcpFileUlsaAnalyserLogs' INTO TABLE enm_ulsa_spectrum_analyser_logs" )
    or die "Failed to load new data from '$bcpFileUlsaAnalyserLogs' file to 'enm_ulsa_spectrum_analyser_logs' table" . $dbh->errstr;
}

1;

Answer 1

I would say that the problem is your

my $bcpFileUlsaAnalyserLogs = getBcpFilename( "enm_ulsa_spectrum_analyser_logs " )

I doubt if the file name really has a space at the end!

I'm surprised that your dbDo or at least your

or die "Failed to load new data ... "

didn't catch this and report it. I don't like wrapping DBI code inside subroutines, especially when I can't see those subroutines.

Note also that you should be using placeholders

dbDo( $dbh, sprintf("DELETE FROM enm_ulsa_spectrum_analyser_logs  WHERE siteid = %d AND time BETWEEN '%s' AND '%s'",
$self->{'siteId'}, $self->{'r_instrDataEvent'}->{'ulsaStats'}->[0]->{'epochtime'},$self->{'r_instrDataEvent'}->{'ulsaStats'}->[$#{$self->{'r_instrDataEvent'}->{'ulsaStats'}}]->{'epochtime'}))
or die "Failed to delete from enm_ulsa_spectrum_analyser_logs" . $dbh->errstr;#'

would be much better as

my $sth = $dbh->prepare(<<END_SQL);
DELETE FROM enm_ulsa_spectrum_analyser_logs
WHERE siteid = ? AND time BETWEEN ? AND ?
END_SQL

my $ulsa_stats = $self->{r_instrDataEvent}{ulsaStats};

$sth->execute(
    $self->{siteId},
    $ulsa_stats->[0]{epochtime}, 
    $ulsa_stats->[-1]{epochtime}
) or die "Failed to delete from enm_ulsa_spectrum_analyser_logs: " . $dbh->errstr;