[m-rev.] for review: make gator copy benchmark program sources automatically

[Samrith UONG]

For review by Julien:

Estimated hours taken: 4

extras/gator/gator:
	Copy the source code for all the benchmark programs to all
	the hosts automatically.  It must already exist on (at least)
	one host.

	Print some more verbose output so that the program gives useful
	feedback to the user without having to pass gator the -v flag.

	At the end of every generation, print the top 10 (or as
	specified by the user with the -n flag) individuals.

extras/gator/README:
	Document the fact that it automatically copies files over,
	rather than requiring the user to do it manually.

Index: README
===================================================================
RCS file: /home/mercury/mercury1/repository/mercury/extras/gator/README,v
retrieving revision 1.2
diff -u -r1.2 README
--- README	17 Feb 2006 01:45:10 -0000	1.2
+++ README	17 Feb 2006 04:26:22 -0000
@@ -136,11 +136,12 @@

 For example, suppose we want to change the program being optimised.
 The new program is in "benchmarks/progs/ray/proj.m", rather than
-"mercury/samples/hello.m".  The first thing that needs to be done is to
-copy the program's source code to each of the hosts.  Then, the
-gator.conf file needs to be modified.  The new file is similar to the
-example given above, except that the $benchmarks1 and $benchmarks2
-directories are different:
+"mercury/samples/hello.m".  The first thing that needs to be done is
+to copy the program's source code to any one of the hosts.  gator will
+then find it and copy it to the other hosts.  Then, the gator.conf
+file needs to be modified.  The new file is similar to the example
+given above, except that the $benchmarks1 and $benchmarks2 directories
+are different:

 	benchmarks1=/home/jupiter/samrith/benchmarks
 	benchmarks2=/home/saturn/samrith/benchmarks
Index: gator
===================================================================
RCS file: /home/mercury/mercury1/repository/mercury/extras/gator/gator,v
retrieving revision 1.1.1.1
diff -u -r1.1.1.1 gator
--- gator	13 Feb 2006 01:42:02 -0000	1.1.1.1
+++ gator	17 Feb 2006 04:15:57 -0000
@@ -28,19 +28,21 @@
 #

 prog=`basename "$0"`
-usage="usage: $prog [-g generation] [-k] [-v]"
+usage="usage: $prog [-g generation] [-k] [-n count] [-v]"

-while getopts g:kv f
+while getopts g:kn:v f
 do
 	case $f in
 	g)	generation="$OPTARG";;
 	k)	kill=true;;
+	n)	count="$OPTARG";;
 	v)	set -x;;
 	\?)	echo "$usage" >&2; exit 1;;
 esac
 done
 shift `expr $OPTIND - 1`

+echo '==> Reading gator.conf' >&2
 . gator.conf || exit 1

 #
@@ -59,13 +61,64 @@
 	while [ $i -le $num_hosts ]
 	do
 		eval "host=\${host$i}"
+		printf "==> Killing evaluate on %s\n" $host >&2
 		ssh "$host" pkill evaluate
 		i=`expr $i + 1`
 	done

+	echo '==> Killing ssh-agent' >&2
+	ssh-agent -k
+
+	echo '==> Killing gator' >&2
 	exec pkill gator
 fi

+tmp="${TMPDIR:-/tmp}"/"$prog"$$
+rm -rf "$tmp" && mkdir "$tmp" || exit 1
+
+trap 'ssh-agent -k; rm -rf "$tmp"; exit' 0 1 2 3 15
+
+i=1
+while [ $i -le $num_hosts ]
+do
+	eval "host=\${host$i}"
+	eval "benchmarks=\${benchmarks$i}"
+
+	if ssh "$host" [ -d "$benchmarks" ]
+	then
+		printf "==> Copying sources from %s:%s\n" "$host" \
+		    "$benchmarks" >&2
+
+		ssh "$host" '(cd '"$benchmarks"' && tar -cf - .) | bzip2' \
+		    >"$tmp"/benchmarks.tar.bz2
+		break
+	fi
+
+	i=`expr $i + 1`
+done
+
+if [ ! -f "$tmp"/benchmarks.tar.bz2 ]
+then
+	echo '==> Error: couldn'\''t find benchmark program sources' >&2
+	exit 1
+fi
+
+i=1
+while [ $i -le $num_hosts ]
+do
+	eval "host=\${host$i}"
+	eval "benchmarks=\${benchmarks$i}"
+
+	printf "==> Copying sources to %s:%s\n" "$host" "$benchmarks" >&2
+
+	ssh "$host" rm -rf "$benchmarks"
+	ssh "$host" mkdir "$benchmarks"
+	ssh "$host" 'bzip2 -dc - | (cd '"$benchmarks"' && tar -xf -)' \
+	    <"$tmp"/benchmarks.tar.bz2
+
+	i=`expr $i + 1`
+done
+
 mmc --make evolve || exit 1

 generation=${generation:-1}
@@ -77,10 +130,8 @@
 	# evaluating a genotype.
 	#

-	fifo="${TMPDIR:-/tmp}"/"$prog"$$
-	rm -rf "$fifo" || exit 1
-	mkfifo "$fifo"
-	trap 'cd /; /bin/rm -f "$fifo"; exit' 0 1 2 3 15
+	fifo="$tmp"/fifo
+	rm -rf "$fifo" && mkfifo "$fifo" || exit 1

 	#
 	# Note that throughout the program, the shell variable $i is
@@ -116,6 +167,9 @@
 		eval "benchmarks=\${benchmarks$i}"
 		eval "path=\${path$i}"

+		printf "==> Connecting to %s (%d/%d = %d%%)\n" $host $j \
+		    $num_genotypes `expr 100 \* $j / $num_genotypes` >&2
+
 		(
 			sed -n ${j}p <"$genotypes" | ssh "$host" nice -n 19 \
 			    "$workspace"/evaluate -b "$benchmarks" -p "$path" \
@@ -164,11 +218,21 @@
 	next_genotypes=generations/$next_generation/genotypes
 	ladder=generations/$generation/ladder

+	printf "==> Next generation: %d\n" $next_generation >&2
+
 	mkdir generations/$next_generation
 	./evolve -g "$genotypes" -h "$next_genotypes" -l "$ladder" \
 	    -p "$phenotypes" || exit 1

 	generation="$next_generation"
+
+	#
+	# Print the top $count individuals in the history of the evolution.
+	#
+
+	echo "==> Current top %d\n" $n >&2
+	sort -rnk3 generations/*/ladder | grep -v '[0-9]e-' | head -n $count |
+	    sed -e 's/[ 	]*$//'
 done

 # NOTREACHED

--------------------------------------------------------------------------
mercury-reviews mailing list
post:  mercury-reviews at cs.mu.oz.au
administrative address: owner-mercury-reviews at cs.mu.oz.au
unsubscribe: Address: mercury-reviews-request at cs.mu.oz.au Message: unsubscribe
subscribe:   Address: mercury-reviews-request at cs.mu.oz.au Message: subscribe
--------------------------------------------------------------------------