#!/usr/bin/perl
use LWP::Simple;
require "subroutin.pl";
$ref_file = "Ref.txt";
#------------- Get PubMed Records ----------------------------------
$key = $ARGV[0];
$type = $ARGV[1];
$outfile = $ARGV[2];
$utils = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils";
$db  = "Pubmed";
$report = "MEDLINE";
$time1 = time - 5*3600; 
@date1 =  gmtime($time1); # Get present EST time ( GMT - Time Lag)
$time2 = $time1 - 14*24*3600; # Two Weeks before from $time1
@date2 =  gmtime($time2); 
$year1 = $date1[5]+1900;
$month1 = $date1[4]+1;
$day1 = $date1[3];
$year2 = $date2[5]+1900;
$month2 = $date2[4]+1;
$day2 = $date2[3];
$day_from = "$year2/$month2/$day2";
$day_to = "$year1/$month1/$day1";
print "Download 10,000 records between $day_from and $day_to.\n";
#--------- Query ------------------
#Key is period  yyyy/mm/dd:yyyy/mm/dd
#To get background word distribution 
($key1,$key2) = split(/:/,$key);
$query  = "(\"$day_from\"[EDat]%20:%20\"$day_to\"[EDat])%20AND%20hasabstract[text]";
#--------- Get Key --------------------
$esearch = "$utils/esearch.fcgi?db=$db&retmax=1&usehistory=y&term=$query";
$esearch_result = get($esearch);
$esearch_result =~ m|<Count>(\d+)</Count>.*<QueryKey>(\d+)</QueryKey>.*<WebEnv>(\S+)</WebEnv>|s;

$Count    = $1;
$QueryKey = $2;
$WebEnv   = $3;

print "Count = $Count\n";
$retstart;
$retmax=10000;
#$retmax=10;
#---------- Download and Save Records -------------
$efetch = "$utils/efetch.fcgi?rettype=$report&retmode=text&retstart=$retstart&retmax=$retmax&db=$db&query_key=$QueryKey&WebEnv=$WebEnv";
$efetch_result = get($efetch);
$date =~ tr/\///d;
open OUT, ">temp/$ref_file";
print OUT "$efetch_result\n";
close OUT;
print "Download was completed.\n";
#----------------- Make WordBG.txt -----------------------
open IN, "temp/$ref_file";
$idf_cutoff = log(1);
$num_records_cutoff = 2;
$tfidf_cutoff = 0;
$DocNum = 0;
while (<IN>){
	chomp;
	if(/^PMID-\s+(\d+)/){
		$pmid = $1;
		push(@pmids,$pmid);
	}
	if(/^AB\s+\-(.+)/){
		$DocNum++;
		%WordFlag = undef;
		$text = $1;
		$flag = "AB";
	}
	elsif(/^[A-Z]+/){
		$flag = "";
	}
	if($flag eq "AB"){ 
		if(!$text){
			$text = $_;
		}
		@words = &TokenizeText($text);
		foreach $word (@words){
			$word = &WordTrim($word);
			$tmp = &WordCheck($word);
			if($word and !$tmp){
				$count{$pmid}->{$word}++;
				$count{$pmid}->{'ALL'}++;
				if(!$WordFlag{$word}){
					$WordFlag{$word} = 1;
					$DocCount{$word}++;
				}
			}
		}
		$text = "";
	}
}
close IN;
print "Reading records was completed\n";
@words = keys(%DocCount);
for($i=0;$i<@words;$i++){
	$tf{$words[$i]} = 0;
	for($j=0;$j<@pmids;$j++){
		next unless $count{$pmids[$j]}->{$words[$i]};
		$tmp = $count{$pmids[$j]}->{$words[$i]}/$count{$pmids[$j]}->{'ALL'};
		$tf{$words[$i]} = $tmp if $tmp > $tf{$words[$i]};
	}
}
while (($word,$count) = each(%DocCount)){
	if($count < $num_records_cutoff){
		next;
	}
	$idf{$word} = log($DocNum) - log($count);
	$tfidf{$word} = $tf{$word}*$idf{$word};
}
@words = keys(%idf);
@words = sort{$tfidf{$b}<=>$tfidf{$a}}@words;
open OUT, ">WordBG.txt";
foreach $word (@words){
	print OUT "$word\t$DocCount{$word}\t$idf{$word}\t$tfidf{$word}\n";
	$c++;
}
close OUT;
print "WordBG.txt file was created.\n";
exit;
