#!/usr/bin/perl -w
my $TASK_DIR="/usr/";
my $VER = "1.52";
# Brian Carrier [carrier@atstake.com]
# Copyright (c) 2002 Brian Carrier, @stake Inc.  All rights reserved
#
# sorter is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# sorter is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with mactime; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR ANY PARTICULAR PURPOSE.
#
# IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, LOSS OF USE, DATA, OR PROFITS OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#


use strict;

$ENV{PATH} = '';

my $BIN_DIR = "$TASK_DIR/bin/";
my $SHARE_DIR = "$TASK_DIR/share/sorter/";

my $TASK_FLS = "${BIN_DIR}fls";
my $TASK_ILS = "${BIN_DIR}ils";
my $TASK_ICAT = "${BIN_DIR}icat";
my $TASK_FILE = "${BIN_DIR}file";
my $TASK_MD5 = "${BIN_DIR}md5";
my $TASK_SHA1 = "${BIN_DIR}sha1";

my $MIS_NAME = "/mismatch_ext.txt";
my $UNK_NAME = "/unknown_cat.txt";


# CONSTANTS
my $DEL_ALLOC = 0;	# Allocated File
my $DEL_DEL = 1;	# Deleted File
my $DEL_ILS = 2;	# Deleted file from ILS, which means the name is not there


# Figure out how to allow no extension as a legit extension

sub usage {
print <<EOF;

------  WARNING:  BETA VERSION  -----

sorter [-d dir] [-ehilsvV] [-md5] [-sha1] [-c config] [-C config] [-n nist_dir] -f fstype image1 [image2 ...]
	-e: Perform extension checks only (no category index files)
	-h: HTML Format
	-i: Perform category indexing only (no extension checks)
	-l: List index to STDOUT (no files are ever written)
	-md5: Print the MD5 value with the index output
	-s: Save files to category directories
	-sha1: Print the SHA-1 value with the index output
	-v: verbose debugging output
	-V: print version information
	-c config: specify a config file to use (in addition to default files)
	   NOTE: This config file has priority over default files
	-C config: specify the ONLY config file to use
	-d dir: Save category index files in the specified directory
	-f fstype: file system type (TASK types) of image
	-n nist_dir: directory where NIST NSRL is located
	image: image to analyze
EOF
	exit(1);
};

sub version {
	print "TASK ver $VER\n";
}

# Globals
my %file_to_cat;
my %file_to_ext = (
	NOT_USED => [","]
);
my %cat_handle;

# Argument variables
my $HTML = 0;
my $LIST = 0;
my $SAVE = 0;
my $VERBOSE = 0;
my $DO_MD5 = 0;
my $DO_SHA1 = 0;
my $ALL_CONFIGS = 1;
my $DO_INDEX = 1;
my $DO_EXT = 1;

my $CONFIG = "";
my $DIR = "";
my $FSTYPE = "";
my $NSRL = "";
my $PLATFORM = "";


my $cur_img;
my $img_shrt;
my $TEMP_FILE;

usage() if (scalar @ARGV == 0);

# Read the arguments
while(($_ = $ARGV[0]) =~ /^-(.)(.*)/) {

	# config file
	if (/^-c$/) {
		if ($ALL_CONFIGS == 0) {
			print "-c cannot be used with -C\n";
			exit(1);
		}
		shift(@ARGV);
		if (defined $ARGV[0]) {
			$CONFIG = $ARGV[0];
		} else {
			print "-c requires config file argument\n";
			usage();
		} 
		unless (-e $CONFIG) {
			print "Config file $CONFIG does not exist\n";
			usage();
		}
	}
	elsif (/^-C$/) {
		if ($CONFIG ne "") {
			print "-C cannot be used with -c\n";
			exit(1);
		}
		shift(@ARGV);
		if (defined $ARGV[0]) {
			$CONFIG = $ARGV[0];
		} else {
			print "-C requires config file argument\n";
			usage();
		} 
		unless (-e $CONFIG) {
			print "Config file $CONFIG does not exist\n";
			usage();
		}
		$ALL_CONFIGS = 0;
	}
	elsif (/^-d$/) {
		shift(@ARGV);
		if (defined $ARGV[0]) {
			$DIR = $ARGV[0];
		} else {
			print "-d requires directory name\n";
			usage();
		} 
		unless (-d $DIR) {
			print "Directory $DIR does not exist\n";
			usage();
		}
	}
	elsif (/^-e$/) {
		$DO_INDEX = 0;
	}
	elsif (/^-f$/) {
		shift(@ARGV);
		if (defined $ARGV[0]) {
			$FSTYPE = $ARGV[0];
		} else {
			print "-f requires file system type\n";
			usage();
		} 
	}
	elsif (/^-h$/) {
		$HTML = 1;
		print "HTML Support is not complete\n";
		exit(1);
	}
	elsif (/^-i$/) {
		$DO_EXT = 0;
	}
	elsif (/^-l$/) {
		$LIST = 1;
	}
	elsif (/^-md5$/) {
		$DO_MD5 = 1;
	}
	elsif (/^-n$/) {
		shift(@ARGV);
		if (defined $ARGV[0]) {
			$NSRL = $ARGV[0];
		} else {
			print "-n requires directory name\n";
			usage();
		} 
		unless (-d $NSRL) {
			print "Directory $NSRL does not exist\n";
			usage();
		}
		unless (-e "${NSRL}/NSRLFile.txt") {
			print "NSRL Database file missing (${NSRL}/NSRLFile.txt)\n";
			usage();
		}
		$DO_SHA1 = 1;

		print "NIST NSRL Not yet implemented\n";
		exit(1);
	}
	elsif (/^-sha1$/) {
		$DO_SHA1 = 1;
	}
	elsif (/^-s$/) {
		$SAVE = 1;
	}
	elsif (/^-V$/) {
		version();
		exit(0);
	}
	elsif (/^-v$/) {
		$VERBOSE = 1;
	}
	else {
		print "Unknown option: $_\n";
		usage();
	}
	shift(@ARGV);
}

if (scalar @ARGV == 0) {
	print "Missing image argument\n";
	usage();
}


# verify that the correct arguments were given
check_args();


# Verify that the binaries are there
check_execs();


# Set the $PLATFORM variable based on $FSTYPE
set_platform();


# Read the config file
if ($ALL_CONFIGS == 1) {
	read_config("${SHARE_DIR}default.sort")
	  if (-e "$SHARE_DIR/default.sort");

	read_config("${SHARE_DIR}${PLATFORM}.sort")
	  if (($PLATFORM ne "") && (-e "${SHARE_DIR}${PLATFORM}.sort"));

	read_config("${SHARE_DIR}${PLATFORM}.lcl.sort")
	  if (($PLATFORM ne "") && (-e "${SHARE_DIR}${PLATFORM}.lcl.sort"));
}

read_config($CONFIG) if ($CONFIG ne "");


# any config data?
if ((scalar (keys %file_to_cat) == 0) && ($DO_INDEX == 1) && ($DO_EXT == 0)) {
	print "Error: Empty config files\n";
	exit (1);
}

if ((scalar (keys %file_to_ext) == 0) && ($DO_EXT == 1) && ($DO_INDEX == 0)) {
	print "Error: No defined extensions\n";
	exit (1);
}

# Open the file handles
open_files() if ($LIST == 0);

while (my $IMG = $ARGV[0]) {
	unless ((-e $IMG) || (-l $IMG)) {
		print "Image file not found: $IMG\n";
		exit(1);
	}

	$cur_img = $IMG;

	$img_shrt = $cur_img;
	$img_shrt = substr ($cur_img, rindex ($cur_img, '/') + 1) 
	  if ($cur_img =~ /\//);

	$TEMP_FILE = "${DIR}/.sorter-$img_shrt-$$-";

	analyze_img ();
	shift(@ARGV);
}

close_files() if ($LIST == 0);

exit(0);


#########################################################################
# subroutines


# Analyze one image.  This function calls 'fls' and 'ils', parses the
# output and then calls analyze_file for each file
#
# No arguments
#
sub analyze_img {
	my $cnt = 0;
	my $inode;
	my $path;

	# Process the allocated files in the image
	print "Loading Allocated File Listing\n" if ($LIST == 0);

	my @out = `$TASK_FLS -f $FSTYPE -rp $cur_img`;
	my $total = $#out;
	print "Processing $total Allocated Files\n" if ($LIST == 0);

	my $prev = 0;
	foreach (@out) {
		my $del;

		# Print the status
		if ((++$cnt % 1000) == 0) {
			my $cur = int(100 * ($cnt / $total));
			if ($cur > $prev + 1) {
				print "$cur%,";
				$prev = $cur;
			}
		}

		# Extract the file name and inode, skip if it is a directory
		if (/^([\w\-])\/[\w\-]\s+(\*?)\s*([\d\-]+)[\(\)\w]*:\s+(.*)\s*$/) {
			next if ($1 eq "d");
			$inode = $3;
			$path = $4;
			$del = ($2 eq '*') ? $DEL_DEL : $DEL_ALLOC;
		}
		else {
			print "Error Parsing Output: $_";
			next;
		}

		analyze_file ($path, $inode, $del);
	}
	print "100%\n";

	# Process the unallocated files in the image
	$cnt = 0;
	print "Loading Unallocated File Listing\n" if ($LIST == 0);
	@out = `$TASK_ILS -f $FSTYPE -m $cur_img`;
	$total = $#out;
	print "Processing $total Unallocated Files\n" if ($LIST == 0);

	$prev = 0;
	foreach (@out) {
		my $null;
		my $size;

		# Print the status
		if ((++$cnt % 500) == 0) {
			my $cur = int(100 * ($cnt / $total));
			if ($cur > $prev + 1) {
				print "$cur%,";
				$prev = $cur;
			}
		}

		# Extract the file name and inode, skip if it is a directory
		($null,$path,$null,$inode,$null,$null,$null,$null,$null,$null,
		  $size,$null) = split ('\|', $_);

		next unless ((defined $inode) && (defined $path) && (defined $size)
		   && ($size =~ /\d+/) && ($size != 0));

		analyze_file ($path, $inode, $DEL_ILS);
	}
	print "100%\n";
}


# Process one file
#
# Arguments are the name of the file, the inode number of the file,
# and the deletion status ($DEL_*)
sub analyze_file {
	if (scalar (@_) != 3) {
		print "Incorrect Number of Arguments for analyze_file\n";
		return;
	}

	my $path = shift;
	my $inode = shift;
	my $del = shift;

	my $sha1 = "";
	my $md5 = "";
	my $file;
	my $mismatch = 0;

	# The FAT full path has the short name in parenths, so 
	# take them off first
	if (($path =~ /\)$/) && ($FSTYPE =~ /fat/)) {
		$path = substr ($path, 0, rindex ($path, '(') - 1);
	}

	# Get the hash values and file type
	if (($LIST) || (($SAVE == 0) && ($DO_MD5 == 0) && ($DO_SHA1 == 0))) {
		$file = `$TASK_ICAT -f $FSTYPE $cur_img $inode | $TASK_FILE -b -`;
		chomp $file;
		if ($DO_SHA1 == 1) {
			$sha1 = `$TASK_ICAT -f $FSTYPE $cur_img $inode | $TASK_SHA1`;
			chomp $sha1;
		}
		if ($DO_MD5 == 1) {
			$sha1 = `$TASK_ICAT -f $FSTYPE $cur_img $inode | $TASK_MD5`;
			chomp $md5;
		}
	}
	else {
		`$TASK_ICAT -f $FSTYPE $cur_img $inode > ${TEMP_FILE}$inode`;
		$file = `$TASK_FILE -b ${TEMP_FILE}$inode`;
		chomp $file;
		if ($DO_SHA1 == 1) {
			$sha1 = `$TASK_SHA1 ${TEMP_FILE}$inode`;
			$sha1 = $1 if ($sha1 =~ /^([A-Fa-f0-9]+)\s+.*$/);
		}
		if ($DO_MD5 == 1) {
			$md5 = `$TASK_MD5 ${TEMP_FILE}$inode`;
			$md5 = $1 if ($md5 =~ /^([A-Fa-f0-9]+)\s+.*$/);
		}

		unlink ("${TEMP_FILE}$inode") if ($SAVE == 0);
	}

	if ($file eq "emtpy") {
		unlink ("${TEMP_FILE}$inode") if ($SAVE == 1);
		return;
	}


	#
	# Extension Stuff
	#

	my $ext = "";
	# Is there an extension on this file?
	my $ext_off = rindex ($path, ".");
	# Some sanity checks to verify that the '.' is after the '/' and
	# add one so that we don't process /.asd as an extension
	if (($ext_off != -1) && ($ext_off > (rindex ($path, "/") + 1))) {
		$ext = substr ($path, $ext_off + 1);
		$ext =~ tr/[A-Z]/[a-z]/;
	}

	$path .= " (deleted)" if ($del == $DEL_DEL); 

	if ($VERBOSE) {
		print "File $path (ext: $ext)\n";
		print "File Output: $file\n";
	}

	# Check the extension if it exists and we do not have 'ils' output
	if (($DO_EXT == 1) && ($ext ne "") && ($del != $DEL_ILS) && 
	  ($file ne 'data')) {

		my $found = 0;

		# cycle through the known file keywords that have a known ext
		foreach my $ext_kw (keys %file_to_ext) {

			print "Trying Extension Keyword: $ext_kw\n" if ($VERBOSE);

			# is this the 'file' category?
			if ($file =~ /$ext_kw/i) {

				print "Found Extension Keyword\n" if ($VERBOSE);

				# we found at least one set of extensions that matches
				# this file type, so set the mismatch to 1 and if we
				# find this extension we will set it to 0, otherwise
				# it will be considered a mismatch
				$mismatch = 1;

				$ext =~ tr/[A-Z]/[a-z/;

				# cycle through each possible extension for this type
				foreach my $cat_ext (@{$file_to_ext{$ext_kw}}) {
					print "Comparing ext with $cat_ext\n" if ($VERBOSE);
					if ($cat_ext eq $ext) {
						print "Found ext\n" if ($VERBOSE);
						$mismatch = 0;
						$found = 1;
						last;
					}
				}
			}

			# If we have found the extension, then get out of the loop
			last if ($found == 1);
		}
	}
	#
	# Lookup in database at this point if we had one
	#
	# $NSRL

	#
	# Get the category
	#

	my $save_name = "";

	my $cat = ""; 
	if ($DO_INDEX) {
		# is this a category we want to save data about?  
		foreach my $cat_kw (keys %file_to_cat) {
			if ($file =~ /$cat_kw/i) {
				$cat = $file_to_cat{$cat_kw};

				# Are we going to save this to a directory? 
				if ($SAVE == 1) {
					my $save_dir = "${DIR}/${cat}";
					mkdir ($save_dir, 0775) unless (-d $save_dir);

					if ($ext eq "") {
						$save_name = "${img_shrt}-${inode}";
					} else {
						$save_name = "${img_shrt}-${inode}.${ext}";
					}
					rename ("${TEMP_FILE}$inode", 
					  "${save_dir}/${save_name}");
				}

				last;
			}
		}
	}
	unlink ("${TEMP_FILE}$inode") 
	  if (($SAVE == 1) && (-e "${TEMP_FILE}$inode"));

	######################################################
	#
	# Print the results
	#
	######################################################


	# @@@ Add HTML
	# add what the extension is configured to mean as well

	if ($LIST == 1) {
		print "Category: $cat\n" if ($cat ne "");
		print "$path\n".
		  "$file\n".
		  "--- Extension Mismatch! ---\n" if ($mismatch == 1);

		print "Image: $cur_img  Inode: $inode\n";
		print "SHA-1: $sha1\n" if ($DO_SHA1 == 1);
		print "MD5: $md5\n" if ($DO_MD5 == 1);

		print "\n";
	}
	elsif ($cat ne "") {
		my $tmphandle = $cat_handle{$cat};

		print $tmphandle "$path\n";
		print $tmphandle "  $file\n";
		print $tmphandle "  --- Extension Mismatch! ---\n" 
		  if ($mismatch == 1);

		print $tmphandle "  Image: $cur_img  Inode: $inode\n";
		print $tmphandle "  SHA-1: $sha1\n" if ($DO_SHA1 == 1);
		print $tmphandle "  MD5: $md5\n" if ($DO_MD5 == 1);
		print $tmphandle "  Saved to: ${cat}/${save_name}\n"
		  if ($SAVE == 1);

		print $tmphandle "\n";

	}
	# if we didn't find a category (and we were looking for one)
	# then put it in unknown
	elsif ($DO_INDEX == 1) {
		if (($file ne "data") && ($file ne "empty")) {
			print UNKNOWN "$path\n";
			print UNKNOWN "  (Ext: $ext) $file\n";
			print UNKNOWN "  Image: $cur_img  Inode: $inode\n\n";
		}
	}

	if (($mismatch == 1) && ($LIST == 0)) {
		print MISMATCH "$path\n";
		print MISMATCH "  (Ext: $ext) $file\n";
		print MISMATCH "  Image: $cur_img  Inode: $inode\n";
		print MISMATCH "  SHA-1: $sha1\n" if ($DO_SHA1 == 1);
		print MISMATCH "  MD5: $md5\n" if ($DO_MD5 == 1);
		print MISMATCH "\n";
	}
	return;
}


# Read the config files
sub read_config {
	my $config = shift;

	open(CONFIG, "$config") or die "Can't open $config";

	print "Reading $config\n" if ($VERBOSE);

	while (<CONFIG>) {
		next if ((/^\#/) || (/^\s+$/));

		# category definition
		# category name key_words
		if (/^\s*category\s+([\w\d]+)\s+(.*?)\s*$/) {
			my $kw = $2;
			my $cat = $1;

			# Make lowercase
			$cat =~ tr/[A-Z]/[a-z]/;

			# mismatch is a reserved word because we make a 
			# mismatch.txt file for all mismatched extensions
			if ($cat eq "mismatch_ext") {
				print "Invalid Category Name: $cat (Reserved)\n";
				exit(1);
			}

			# do a sanity check to see if we are overriding a 
			# category that already existed for this file type
			if ((exists $file_to_cat{$kw}) &&
			  ($file_to_cat{$kw} ne $cat)) {
				print "Warning: overriding category $file_to_cat{$kw} with $cat for key words: $kw\n";
			}

			$file_to_cat{$kw} = $cat;
			print "Adding Category: $cat   File Keywords: $kw\n" if ($VERBOSE);
		}

		# extention defn
		# ext ext1,ext2, key_words
		elsif (/^\s*ext\s+([\w\d\,]+)\s+(.*?)\s*$/) {
			my $ext = $1;
			my $kw = $2;

			# Make lowercase 
			$ext =~ tr/[A-Z]/[a-z]/;
	
			# If there are already some extensions, then we will just
			# extend them
			if (exists $file_to_ext{$kw}) {

				# We could just do a push, but then we risk having
				# duplicate entries, which will waste time later
				foreach my $e1 (split (/,/, $ext)) {
					my $exists = 0;
					foreach my $e2 (@{$file_to_ext{$kw}}) {
						if ($e1 eq $e2) {
							$exists = 1;
							last;
						}
					}
					push @{$file_to_ext{$kw}}, $e1 if ($exists == 0);
				}
				print "Adding Extensions: $ext   File Keywords: $kw\n" 
				  if ($VERBOSE);

			} else {
				$file_to_ext{$kw} = [split (/,/, $ext)];
				print "New Extensions: $ext   File Keywords: $kw\n" 
				  if ($VERBOSE);
			}
		}
		else {
			print "Invalid line in $config:$.\n"; 
			exit(1);
		}
	}

	close (CONFIG);
};


# Open the summary files into an array of handles
sub open_files {
	return if ($LIST == 1);


	if ($DO_EXT == 1) {
		open(MISMATCH, ">${DIR}${MIS_NAME}") 
		  or die "Can't open ${DIR}${MIS_NAME}";
	}

	if ($DO_INDEX == 1) {
		open(UNKNOWN, ">${DIR}${UNK_NAME}")
		  or die "Can't open ${DIR}${UNK_NAME}";

		foreach my $cat (values %file_to_cat) {
			next if (exists $cat_handle{$cat});

			open($cat_handle{$cat}, ">${DIR}/${cat}.txt") 
			  or die "Can't open ${cat}.txt";
		}
	}
};


# Close the output summary files and remove them if they have
# a size of 0
sub close_files {
	return if ($LIST == 1);


	if ($DO_EXT == 1) {
		close (MISMATCH);
		unlink "${DIR}${MIS_NAME}" if (-z "${DIR}${MIS_NAME}");
	}

	if ($DO_INDEX == 1) {
		close (UNKNOWN);
		unlink "${DIR}${UNK_NAME}" if (-z "${DIR}${UNK_NAME}");

		foreach my $cat (keys %cat_handle) {
			close ($cat_handle{$cat});
			unlink "${DIR}/${cat}.txt" if (-z "${DIR}/${cat}.txt");
		}
	}
};

sub check_execs {
	unless (-e "$TASK_FLS") {
		print "Missing TASK fls executable: $TASK_FLS\n";
		exit(1);
	}

	unless (-e "$TASK_ILS") {
		print "Missing TASK ils executable: $TASK_ILS\n";
		exit(1);
	}

	unless (-e "$TASK_FILE") {
		print "Missing TASK file executable: $TASK_FILE\n";
		exit(1);
	}

	unless (-e "$TASK_ICAT") {
		print "Missing TASK icat executable: $TASK_ICAT\n";
		exit(1);
	}

	if ($DO_SHA1 == 1) {
		unless (-e "$TASK_SHA1") {
			print "Missing TASK sha1 executable: $TASK_SHA1\n";
			exit(1);
		}
	}

	if ($DO_MD5 == 1) {
		unless (-e "$TASK_MD5") {
			print "Missing TASK md5 executable: $TASK_MD5\n";
			exit(1);
		}
	}
};


# Set the $PLATFORM value from $FSTYPE
sub set_platform {

	if (($FSTYPE eq "ntfs") ||
	  ($FSTYPE eq "fat") ||
	  ($FSTYPE eq "fat32") ||
	  ($FSTYPE eq "fat16") ||
	  ($FSTYPE eq "fat12")) {
		$PLATFORM = "windows";
	}
	elsif ($FSTYPE eq "solaris") {
		$PLATFORM = "solaris";
	}
	elsif ($FSTYPE eq "openbsd") {
		$PLATFORM = "openbsd";
	}
	elsif ($FSTYPE eq "freebsd") {
		$PLATFORM = "freebsd";
	}
	elsif (($FSTYPE eq "linux-ext2") ||
	  ($FSTYPE eq "linux-ext2")) {
		$PLATFORM = "linux";
	}
	else {
		print "Unknown file system type: $FSTYPE\n";
		exit(1);
	}

	print "Platform set to: $PLATFORM\n" if ($VERBOSE);
};

sub check_args {
	# Sanity check the arguments
	if ($FSTYPE eq "") {
		print "Missing file system type\n";
		usage();
	}

	elsif (($DIR eq "") && ($LIST == 0)) {
		print "Missing directory location\n";
		usage();
	}

	elsif (($DIR ne "") && ($LIST == 1)) {
		print "Directory (-d) and List (-l) flags cannot be used together\n";
		usage();
	}

	elsif (($SAVE == 1) && ($LIST == 1)) {
		print "Save Files (-s) and List (-l) flags cannot be used together\n";
		usage();
	}
}
