#!/usr/bin/perl
# Process TI and AB separately
#
#
$TrainDoc = $ARGV[1];
$BG_Data = "WordBG.txt";
$MaxTrainData = $ARGV[2];
@slots = qw(AB TI);
$df_cutoff = 0.1; # Cutoff value of document frequency
$tfidf_cutoff_BG = 0.2;
$tfidf_cutoff_TG = 0.05;
$wc_for_zero = 0.2;
$min_word_length = 3;
$class = 4;
require "subroutin.pl";
$repeat = $ARGV[0];
$head = $ARGV[3];
############### Read Negative Words
open IN, "nega_word.txt";
while (<IN>){
	chomp;
	$ignore{$_} = 1;
}
close IN;
$c_ig = 0;
########### Read IDF
$idf_cutoff = log(1/$df_cutoff); 
open IN, "$BG_Data";
while (<IN>){
	chomp;
	split(/\t/);
	next if length($_[0]) < $min_word_length;
	if(!$zero){
		$zero = log($_[1]) + $_[2]; # calc as single record in case of no record
	}
	if($_[2] < $idf_cutoff and !$ignore{$_[0]}){
		#$c_ig++;
		$ignore{$_[0]} = 1;
	}
	elsif(($_[3] > $tfidf_cutoff_BG) and $ignore{$_[0]}){
		$c_ig++;
	}
	elsif(($_[3] > $tfidf_cutoff_BG) and !$ignore{$_[0]}){
		$wordset{$_[0]} = 1;
		$c++;
	}
	$idf{$_[0]} = $_[1];
}
close IN;
print "Wordset: $c, $c_ig\n";
###################################
$NumAll = 0;
@pmids = ();
open IN, "temp/$TrainDoc";
$DocNum = 0;
while (<IN>){
	if($DocNum >= $MaxTrainData){
		last;
	}
	chomp;
	if(/^PMID\-\s+(\d+)/){
		$pmid = $1;
		push(@pmids,$pmid);
		$DocNum++;
	}
	elsif(/^(AB|TI)\s+\-\s+(.+)/){
		$text = $2;
		$flag = $1;
	}
	elsif(/^[A-Z]+/){
		$flag = "";
	}
	if(($flag eq "AB") or ($flag eq "TI")){ # NOT elsif
		if(!$text){
			s/^\s+//;
			$text = $_;
		}
		@words = &TokenizeText($text);
		foreach $word (@words){
			$word = &WordTrim($word);
			$tmp = &WordCheck($word);
			next if $tmp;
			next if length($word) < $min_word_length;
			push(@{$text{$flag}->{$pmid}},$word);
			$word_c{$flag}->{$word} = &AddCount($word_c{$flag}->{$word});
			$word_c_each{$flag}->{$pmid}->{$word} = &AddCount($word_c_each{$flag}->{$pmid}->{$word});
			$word_c{$flag}->{'ALL'} = &AddCount($word_c{$flag}->{'ALL'});
		}
		$text = "";
	}
}
close IN;
########### Calc TF-IDF of Abstract
%tfidf = ();
$total = $word_c{'AB'}->{'ALL'};
@words = keys(%{$word_c{'AB'}});
foreach $word (@words){
	next if $word eq 'ALL';
	if(!$idf{$word}){
		$idf{$word} = $zero;
	}
	if(!$ignore{$word}){
		$tfidf{$word} = $word_c{'AB'}->{$word}*$idf{$word}/$total;
	}
}
while (($word,$val) = each(%tfidf)){
	if($val > $tfidf_cutoff_TG and !$ignore{$word}){
		if(!$wordset{$word}){
			$wordset{$word} = 1;
		}
	}
}
@words = keys(%wordset);
######## Initialize P(c) ####
for($i=0;$i<$class;$i++){
	$pc[$i] = log(1/$class);
}
######## Initialize P(w,c) ####
foreach $slot (@slots){
	$word_c{$slot}->{'ALL'} = log($word_c{$slot}->{'ALL'});
}
for($i=0;$i<@words;$i++){
	## Set pseudecount to each value
	for($j=0;$j<@pmids;$j++){
		foreach $slot (@slots){
			if(!$word_c_each{$slot}->{$pmids[$j]}->{$words[$i]}){
				$word_c_each{$slot}->{$pmids[$j]}->{$words[$i]} = $wc_for_zero;
			}
			$word_c_each{$slot}->{$pmids[$j]}->{$words[$i]} = log($word_c_each{$slot}->{$pmids[$j]}->{$words[$i]});
			if($j<$class){ # use first classes of records for initial value 
				$pwc[$j]->{$slot}->{$words[$i]} = $word_c_each{$slot}->{$pmids[$j]}->{$words[$i]} - $word_c{$slot}->{'ALL'};
			}
		}
	}
}
## Deduce parameter with EM
for($r=0;$r<$repeat;$r++){
	######## E Step ##############
	$pdSum = undef;
	for($i=0;$i<@pmids;$i++){
		for($j=0;$j<$class;$j++){
			$pdc[$j]->{$pmids[$i]} = $pc[$j];
		}
		foreach $slot (@slots){
			for($j=0;$j<@{$text{$slot}->{$pmids[$i]}};$j++){
				$word = $text{$slot}->{$pmids[$i]}->[$j];
				if($wordset{$word}){
					for($k=0;$k<$class;$k++){
						$pdc[$k]->{$pmids[$i]} += $pwc[$k]->{$slot}->{$word};
					}
				}
			}
		}
		$pd{$pmids[$i]} = undef;
		for($j=0;$j<$class;$j++){
			$pd{$pmids[$i]} = &AddLogVal($pd{$pmids[$i]},$pdc[$j]->{$pmids[$i]});
		}
		for($j=0;$j<$class;$j++){
			$pcd[$j]->{$pmids[$i]} = $pdc[$j]->{$pmids[$i]} - $pd{$pmids[$i]};
		}
		$pdSum += $pd{$pmids[$i]};
	}
	print "PDSUM: $r, $pdSum\n"; 
	############## M Step ##############
	# Updata  P(c) value
	$SumSum_pcd = undef;
	for($i=0;$i<$class;$i++){
		$Sum_pcd[$i] = undef;
		for($j=0;$j<@pmids;$j++){
			$Sum_pcd[$i] = &AddLogVal($Sum_pcd[$i],$pcd[$i]->{$pmids[$j]});
			$SumSum_pcd = &AddLogVal($SumSum_pcd,$pcd[$i]->{$pmids[$j]});
		}
	}
	for($i=0;$i<$class;$i++){
		$pc[$i] = $Sum_pcd[$i] - $SumSum_pcd;
	}
	# Update P(w|c) value 
	for($i=0;$i<$class;$i++){
		foreach $slot (@slots){
			$SumSum_pwc[$i]->{$slot} = undef;
			for($j=0;$j<@words;$j++){
				$Sum_pwc[$i]->{$slot}->{$words[$j]} = undef;
				for($k=0;$k<@pmids;$k++){
					$tmp = $word_c_each{$slot}->{$pmids[$k]}->{$words[$j]}+$pcd[$i]->{$pmids[$k]};
					$Sum_pwc[$i]->{$slot}->{$words[$j]} = &AddLogVal($Sum_pwc[$i]->{$slot}->{$words[$j]},$tmp);
					$SumSum_pwc[$i]->{$slot} = &AddLogVal($SumSum_pwc[$i]->{$slot},$tmp);
				}
			}
		}
	}
	for($i=0;$i<$class;$i++){
		foreach $slot (@slots){
			for($j=0;$j<@words;$j++){
				$pwc[$i]->{$slot}->{$words[$j]} = $Sum_pwc[$i]->{$slot}->{$words[$j]} - $SumSum_pwc[$i]->{$slot};
			}
		}
	}
}
#open OUT, ">parameter.txt";
open OUT, ">temp/$head.parameter.txt";
$pc = "PC";
for($i=0;$i<$class;$i++){
	$pc .= "\t$pc[$i]";
}
print OUT "$pc\n";
foreach $slot (@slots){
	print OUT "PWC:$slot\n";
	for($i=0;$i<@words;$i++){
		$word = $words[$i];
		for($j=0;$j<$class;$j++){
			$word .= "\t$pwc[$j]->{$slot}->{$words[$i]}";
		}
		print OUT "$word\n";
	}
}
close OUT;
