一个1-gram实现，网上有个类似的python的，由于要交作业，写了个perl的，娱乐而已。

备份一下。无版权，需要自取。

#!/usr/bin/perl -w

# Attention please!
# This program should only be executed in UNIX-like platform.
# If run in windows, some unexpected problem will appear.
# In most cases, windows text editor will add in some special control byte in your text file.
# They are sheltered by windows and you almost cannot find them.
# What is worse, this is a chinese segmentation program, an unwanted byte will cause disaster!
#===============================================================================
# Introduction:
# The algorithm used is 1-gram.
# Simply, isn't it? But I don't think so...
# 
# Basicly, three extra files is needed.
# 1. The one used for create a dictinary. Default: 199801q.txt
# 2. The input file. Default: input.txt
# 3. The output file. Default: output.txt
# * Program will create a temp file called "config" to speed up itself.
#===============================================================================
# Author. yinxs
# Mail.   [email protected]
# Welcome to exchage ideas.
#===============================================================================

use Encode;
use utf8;
binmode( STDIN,  ':encoding(utf8)' );
binmode( STDOUT, ':encoding(utf8)' );
binmode( STDERR, ':encoding(utf8)' );

# some global variants.
# If using OOP, we could discard them.
my %trainedDict;
my %dictMap;
my %wordMap;

sub biSplit {
	my @words;
	my $str = $_[0];
	my $len = length($str);
	foreach ( 1 .. $len-1 ) {
		push( @words, substr( $str, 0, $_ ) );
		push( @words, substr( $str, $_, $len - $_ ) );
	}
	@words;
}

sub maxPrb {
	my $sentence = $_[0];
	my $len      = length($sentence);
	if ( $len <= 1 ) {
		return &getPrb($sentence);
	}
	my @words      = &biSplit($sentence);
	my $maxThisPrb = 0;
	my $maxThisPair;
        my $i = 0;
	for ( $i = 0 ; $i < scalar @words - 1 ; $i += 2 ) {
		my $thisPrb = 0;
		if ( length( $words[$i] ) > 0 and length( $words[ $i + 1 ] ) > 0 ) {
			my $p1 = 0;
			my $p2 = 0;
			if ( exists( $dictMap{ $words[$i] } ) ) {
				$p1 = $dictMap{ $words[$i] };
			}
			else {
				$p1 = maxPrb( $words[$i] );
			}

			if ( exists( $dictMap{ $words[ $i + 1 ] } ) ) {
				$p2 = $dictMap{ $words[ $i + 1 ] };
			}
			else {
				$p2 = maxPrb( $words[ $i + 1 ] );
			}
			$thisPrb = $p1 * $p2;
		}
		if ( $thisPrb > $maxThisPrb ) {
			$maxThisPrb     = $thisPrb;
			$maxThisPair = $words[$i]."\t".$words[$i+1];
		}
	}
        $joinPrb = &getPrb($sentence);
        if ( ( $joinPrb > $maxThisPrb ) and &isBelong($sentence) ) {
                $dictMap{$sentence} = $joinPrb;
                $wordMap{$sentence} = $sentence;
                return $joinPrb;
        }
        else {
                $dictMap{$sentence} = $maxThisPrb;
                $wordMap{$sentence} = $maxThisPair;
                return $maxThisPrb;
        }
}

sub display {
	my $output = "output.txt";
	open( OUTPUT, "> $output" ) or die "cannot open output.";
	binmode( OUTPUT, ":encoding(utf8)" );
	while(my ($key,$value) = each(%dictMap)) { 
		print OUTPUT "$key $value\n";
	}
        print OUTPUT "\n";
	while(my ($key,$value) = each(%wordMap)) { 
		print OUTPUT "$key is divided into:\t";
		print OUTPUT $value."\n";
	}
	close(OUTPUT);
}

sub dictInit {
	my $dictName = $_[0];
	my $realWord;
        # If there is a config file already, using it.
        # It will simplify the dictionary initialization.
        if(open( CONFIG, "< config" )) {
                print "Read config file successfully.\n";
                while(<CONFIG>) {
                        chomp;
                        my @dicting = split("\t");
                        $trainedDict{$dicting[0]} = $dicting[1];
                }
                close(CONFIG);
                return;
        }
        open( CONFIG, "> config" ) or die "cannot open config file.";
	open( DICTFD, "< $dictName" ) or die "can't open $dictName.";
        binmode( CONFIG, ":encoding(utf8)" );
	binmode( DICTFD, ":encoding(utf8)" );
	while (<DICTFD>) {
		chomp;
		my @wordSlice = split("  ");
                @wordSlice = @wordSlice[1 .. $#wordSlice];
		foreach (@wordSlice) {

                        # I discarded the POS of the word,
                        # this is just a simple demo of segmentation.
			my @mixedSlice = split("/");
			$realWord = $mixedSlice[0];
			if ( exists( $trainedDict{$realWord} ) ) {
				$trainedDict{$realWord}++;
			}
			else {
				$trainedDict{$realWord} = 1;
			}
		}
	}
	close(DICTFD);
        
        # Writting all content of trainedMap into config file.
        # The next time I would not open and read the naive file.
        while(my ($key,$value) = each(%trainedDict)) { 
		print CONFIG "$key\t$value\n";
	}
        close(CONFIG);
}

sub isBelong {
	exists( $trainedDict{ $_[0] } );
}

sub getCnt {
	if ( exists( $trainedDict{ $_[0] } ) ) {
		$trainedDict{ $_[0] };
	}
	else {
		0.5;    # Using for smoothing.
	}
}

sub getPrb {
	&getCnt( $_[0] ) / ( scalar keys %trainedDict );
}

sub input {
	my $input = "input.txt";
	open( INPUT, "< $input" ) or die "cannot open input.";
	binmode( INPUT, ":encoding(utf8)" );
	while (<INPUT>) {
                chomp;
		&maxPrb($_);
	}
	close(INPUT);
}

sub main {
        &dictInit("199801q.txt");
        &input;
        &display;
}
# main function start
&main;